From b0d587814bbae7f892e0c4ced9ab3b4d7753ffa7 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Thu, 6 Aug 2020 09:50:59 -0500 Subject: [PATCH] doc system service layer improvements --- devdocs/docstructure/topic_mapping.dot | 35 ++++ devdocs/docstructure/topics.dot | 41 ++++ .../aggregator/CompositeMarkdownInfo.java | 28 ++- .../nb/api/markdown/aggregator/MDGraph.java | 121 ++++++++++++ .../api/markdown/aggregator/MarkdownDocs.java | 175 ++++++++++-------- .../aggregator/ParsedFrontMatter.java | 34 +++- .../markdown/aggregator/ParsedMarkdown.java | 23 ++- .../api/markdown/types/FrontMatterInfo.java | 17 +- .../nb/api/markdown/types/MarkdownInfo.java | 48 ++++- .../srcmain-topics-a/srcmain-entry1-1.md | 0 .../srcmain-topic2-1/srcmain-entry-2-1-L.md | 0 .../srcmain-topics-a/srcmain-entry1-1.md | 10 + .../srcmain-topic2-1/srcmain-entry-2-1-L.md | 8 + .../markdown/aggregator/MarkdownDocsTest.java | 74 +++++++- 14 files changed, 518 insertions(+), 96 deletions(-) create mode 100644 devdocs/docstructure/topic_mapping.dot create mode 100644 devdocs/docstructure/topics.dot create mode 100644 nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MDGraph.java rename nb-api/src/main/resources/{docs-for-testing-only => docs-for-testing-logical}/srcmain-topics-a/srcmain-entry1-1.md (100%) rename nb-api/src/main/resources/{docs-for-testing-only => docs-for-testing-logical}/srcmain-topics-b/srcmain-topic2-1/srcmain-entry-2-1-L.md (100%) create mode 100644 nb-api/src/main/resources/docs-for-testing-raw/srcmain-topics-a/srcmain-entry1-1.md create mode 100644 nb-api/src/main/resources/docs-for-testing-raw/srcmain-topics-b/srcmain-topic2-1/srcmain-entry-2-1-L.md diff --git a/devdocs/docstructure/topic_mapping.dot b/devdocs/docstructure/topic_mapping.dot new file mode 100644 index 000000000..69f1673fb --- /dev/null +++ b/devdocs/docstructure/topic_mapping.dot @@ -0,0 +1,35 @@ +digraph { + newrank=true + compound=true + node [fontsize = 8,shape = record] + rankdir = LR; + + subgraph cluster0 { + rankdir = LR; + step0[shape=none] + node [fontsize = 8, shape = record] + A0 [label="A|topic:a,(b)"] + B0 [label="B|topic:b,(c)"] + C0 [label="C|topic:c"] + } + + subgraph cluster1 { + node [fontsize = 8, shape = record] + step1[shape=none] + a1 [label="a",shape=oval] + A1 -> a1 [label="topic of"] + A1 -> b1 [label="topic of"] + expr_b1 -> b1 [label="match"] + expr_b1[label="(b)",shape=oval] + + B1 [label="match"] + b1 [label="b", shape=oval] + B1 -> b1 [label="topic of"] + A1 [label="A"] + B1 [label="B|topic:a,(b)"] + C1 [label="C|topic:b,(c)"] + } + + step0 -> step1[ltail=cluster0,lhead=cluster1] + +} \ No newline at end of file diff --git a/devdocs/docstructure/topics.dot b/devdocs/docstructure/topics.dot new file mode 100644 index 000000000..2b9ab7436 --- /dev/null +++ b/devdocs/docstructure/topics.dot @@ -0,0 +1,41 @@ +digraph { + node [fontsize = 8,shape = record] + rankdir = LR; + + subgraph clusterB { + label = "after topic mapping" + node [fontsize = 8,shape = record] + rankdir = LR; + + ap; bp; cp; dp; ep; + ap [label = "A|topics: all-topics|included: cli,time,temp"] + ap -> {bp; cp; dp; ep} + + bp [label = "B|topics:|included: cli"] + cp [label = "C|topics:cli"] + bp -> cp + + dp [label = "D|topics:temp"] + ep [label = "E|topics:time"] + } + + subgraph clusterA { + label = "before topic mapping" + node [fontsize = 8,shape = record] + rankdir = LR; + + a; b; c; d; e; + + a [label = "A|topics: .*,all-topics"] + a -> {b; c; d; e} + + b [label = "B|topics:(cli)"] + c [label = "C|topics:cli"] + b -> c + + d [label = "D|topics:temp"] + e [label = "E|topics:time"] + + } + +} \ No newline at end of file diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/CompositeMarkdownInfo.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/CompositeMarkdownInfo.java index fe07bc960..8fdeab12c 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/CompositeMarkdownInfo.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/CompositeMarkdownInfo.java @@ -4,11 +4,13 @@ import io.nosqlbench.nb.api.markdown.types.FrontMatterInfo; import io.nosqlbench.nb.api.markdown.types.MarkdownInfo; import java.nio.file.Path; +import java.util.Collections; import java.util.LinkedList; import java.util.List; public class CompositeMarkdownInfo implements MarkdownInfo { private List elements = new LinkedList<>(); + private boolean isSorted=false; @Override public Path getPath() { @@ -18,6 +20,10 @@ public class CompositeMarkdownInfo implements MarkdownInfo { @Override public String getBody() { StringBuilder sb = new StringBuilder(); + if (!isSorted) { + Collections.sort(elements); + isSorted=true; + } for (MarkdownInfo element : elements) { sb.append(element.getBody()); } @@ -35,15 +41,31 @@ public class CompositeMarkdownInfo implements MarkdownInfo { } @Override - public MarkdownInfo withTopics(List assigning) { + public CompositeMarkdownInfo withTopics(List assigning) { MarkdownInfo leader = elements.get(0); leader = leader.withTopics(assigning); elements.set(0,leader); return this; } - public CompositeMarkdownInfo add(T element) { - elements.add(element); + public CompositeMarkdownInfo withIncluded(List included) { + MarkdownInfo leader = elements.get(0); + leader = leader.withIncluded(included); + elements.set(0,leader); return this; } + + public CompositeMarkdownInfo add(T element) { + elements.add(element); + isSorted=false; + return this; + } + + @Override + public String toString() { + return "CompositeMarkdownInfo{" + + "elements=" + elements + + ", isSorted=" + isSorted + + '}'; + } } diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MDGraph.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MDGraph.java new file mode 100644 index 000000000..71e627f3d --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MDGraph.java @@ -0,0 +1,121 @@ +package io.nosqlbench.nb.api.markdown.aggregator; + +import io.nosqlbench.nb.api.markdown.types.MarkdownInfo; + +import java.util.*; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class MDGraph { + + private Map> topicsByPattern; + private final Map>> elementsByPattern = new HashMap<>(); + private final Map>> elementsByTopic = new HashMap<>(); + private final List> elements = new LinkedList<>(); + + public void add(MarkdownInfo addingElem) { + + Edge edge = new Edge<>(addingElem); + + elements.add(edge); + + for (String topic : addingElem.getTopics()) { + elementsByTopic.computeIfAbsent(topic, t -> new ArrayList<>()).add(edge); + } + + // Always add elements to the "none" at a minimum + if (addingElem.getTopics().size() == 0) { + elementsByTopic.computeIfAbsent("none", t -> new ArrayList<>()).add(edge); + } + + for (Pattern pattern : addingElem.getTopicGlobs()) { + elementsByPattern.computeIfAbsent(pattern.pattern(), + p -> new ArrayList<>()).add(edge); + } + } + + public List processed() { + + if (topicsByPattern == null) { + topicsByPattern = topicsByPattern(); + } + + LinkedList> resolved = new LinkedList<>(elements); + + ListIterator> iter = resolved.listIterator(); + + while (iter.hasNext()) { + Edge elementEdge = iter.next(); + MarkdownInfo element = elementEdge.get(); + + List topicGlobs = element.getTopicGlobs(); + + if (topicGlobs.size() != 0) { + List> included = new ArrayList<>(); + boolean leafnodes=true; + for (Pattern topicGlob : topicGlobs) { + for (String matchedTopic : topicsByPattern.get(topicGlob.pattern())) { + List> edges = elementsByTopic.get(matchedTopic); + for (Edge edge : edges) { + if (edge.get().getTopicGlobs().size()!=0) { + leafnodes=false; + } + included.add(edge); + } + } + if (leafnodes) { + CompositeMarkdownInfo mdinfo = + new CompositeMarkdownInfo(); + mdinfo.add(element); + for (Edge tEdge : included) { + mdinfo.add(tEdge.get()); + } + // TODO: Add included + MarkdownInfo withTopics = mdinfo.withTopics(element.getTopics()); + elementEdge.set(withTopics); + } else { + // Move this to the end of the list. + iter.remove(); + resolved.addLast(elementEdge); + } + } + } + } + + return resolved.stream().map(Edge::get).collect(Collectors.toList()); + } + + private Map> topicsByPattern() { + Map> tbp = new HashMap<>(); + for (String pattern : this.elementsByPattern.keySet()) { + List matchingTopics = tbp.computeIfAbsent(pattern, p -> new ArrayList<>()); + for (String topic : this.elementsByTopic.keySet()) { + if (Pattern.compile(pattern).matcher(topic).matches()) { + matchingTopics.add(topic); + } + } + } + return tbp; + } + + + /** + * Allow edges to point to mutable vertices + * @param + */ + private final static class Edge { + private T element; + + public Edge(T element) { + this.element = element; + } + + public T get() { + return element; + } + + public void set(T element) { + this.element = element; + } + } +} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocs.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocs.java index 9bc900634..9f0d903ee 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocs.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocs.java @@ -72,95 +72,122 @@ public class MarkdownDocs { ordered.addAll(markdownWithTopicGlobs); ordered.addAll(markdownInfos); - List>> edges = new ArrayList<>(); - List assigning = null; + MDGraph mdgraph = new MDGraph(); + ordered.forEach(mdgraph::add); - for (int i = 0; i < ordered.size()-1; i++) { - MarkdownInfo mdHavingGlobs = ordered.get(i); - List topicGlobs = mdHavingGlobs.getTopicGlobs(); - - // TODO track and warn if a glob doesn't match anything - for (int j = i+1; j < ordered.size(); j++) { - - MarkdownInfo mdHavingTopics = ordered.get(j); - List topics = mdHavingTopics.getTopics(); - - for (Pattern topicGlob : topicGlobs) { - - for (String topic : topics) { - if (topicGlob.matcher(topic).matches()) { - assigning=assigning==null ? new ArrayList<>() : assigning; - assigning.add(topic); - logger.debug("added topic=" + topic + " to " + i + "->" + j + " with " + topicGlob); - } - } - if (assigning!=null) { - assigning.addAll(mdHavingGlobs.getTopics()); - ordered.set(i,mdHavingGlobs.withTopics(assigning)); - logger.debug("assigned new mdinfo"); - } - } - } - } - - int loopsremaining=100; - - // Assign glob topics to non-glob topics that match - -// for (MarkdownInfo parsedMarkdown : markdownInfos) { -// FrontMatterInfo fm = parsedMarkdown.getFrontmatter(); -// Set topics = fm.getTopics(); -// Set newTopics = new HashSet<>(); -// for (String topic : topics) { -// if (isPattern(topic)) { -// Pattern p = Pattern.compile(topic); -// for (String nonGlobTopic : nonGlobTopics) { -// if (p.matcher(nonGlobTopic).matches()) { -// newTopics.add(topic); + return mdgraph.processed(); +// +// +// +// List>> edges = new ArrayList<>(); +// List matchedtopics = null; +// +// for (int i = 0; i < ordered.size()-1; i++) { +// MarkdownInfo mdHavingGlobs = ordered.get(i); +// List topicGlobs = mdHavingGlobs.getTopicGlobs(); +// +// for (Pattern topicGlob : topicGlobs) { +// for (int matchidx = i+1; matchidx < ordered.size(); matchidx++) { +// MarkdownInfo matchableContent = ordered.get(matchidx); +// List matchableTopics = matchableContent.getTopics(); +// for (String matchableTopic : matchableTopics) { +// if (topicGlob.matcher(matchableTopic).matches()) { +// matchedtopics=matchedtopics==null ? new ArrayList<>() : matchedtopics; +// matchedtopics.add(matchableTopic); +// logger.debug("added topic=" + matchableTopic + " to " + i + "->" + matchidx + " with " + topicGlob); // } // } -// } else { -// newTopics.add(topic); +// if (matchedtopics!=null) { +// matchedtopics.addAll(mdHavingGlobs.getTopics()); +// ordered.set(i,mdHavingGlobs.withTopics(matchedtopics)); +// logger.debug("assigned new mdinfo"); +// matchedtopics=null; +// } // } // } -// fm.setTopics(newTopics); -// } // -// // create topic to content map -// HashMap> contentByTopic = new HashMap<>(); -// for (ParsedMarkdown parsedMarkdown : markdownInfos) { -// for (String topic : parsedMarkdown.getFrontmatter().getTopics()) { -// contentByTopic.computeIfAbsent(topic, t -> new ArrayList<>()).add(parsedMarkdown); -// } -// } +// // TODO track and warn if a glob doesn't match anything +// for (int j = i+1; j < ordered.size(); j++) { // -// ListIterator lit = markdownInfos.listIterator(); -// while (lit.hasNext()) { -// MarkdownInfo mif = lit.next(); -// if (mif.hasAggregations()) { -// lit.remove(); -// mif = new CompositeMarkdownInfo().add(mif); -// lit.add(mif); -// } -// } +// MarkdownInfo mdHavingTopics = ordered.get(j); +// List topics = mdHavingTopics.getTopics(); // -// // combine aggregate targets -// for (ParsedMarkdown parsedMarkdown : markdownInfos) { -// List aggregations = parsedMarkdown.getFrontmatter().getAggregations(); -// if (aggregations.size()>0) { -// for (Pattern aggregation : aggregations) { +// for (Pattern topicGlob : topicGlobs) { // +// for (String topic : topics) { +// if (topicGlob.matcher(topic).matches()) { +// matchedtopics=matchedtopics==null ? new ArrayList<>() : matchedtopics; +// matchedtopics.add(topic); +// logger.debug("added topic=" + topic + " to " + i + "->" + j + " with " + topicGlob); +// } +// } +// if (matchedtopics!=null) { +// matchedtopics.addAll(mdHavingGlobs.getTopics()); +// ordered.set(i,mdHavingGlobs.withTopics(matchedtopics)); +// logger.debug("assigned new mdinfo"); +// } // } // } // } // -// // Assign glob topics +// int loopsremaining=100; // -// // Assign content aggregates -// System.out.println("topics: " + topicSets); +// // Assign glob topics to non-glob topics that match // -// aggregated.addAll(markdownInfos); - return aggregated; +//// for (MarkdownInfo parsedMarkdown : markdownInfos) { +//// FrontMatterInfo fm = parsedMarkdown.getFrontmatter(); +//// Set topics = fm.getTopics(); +//// Set newTopics = new HashSet<>(); +//// for (String topic : topics) { +//// if (isPattern(topic)) { +//// Pattern p = Pattern.compile(topic); +//// for (String nonGlobTopic : nonGlobTopics) { +//// if (p.matcher(nonGlobTopic).matches()) { +//// newTopics.add(topic); +//// } +//// } +//// } else { +//// newTopics.add(topic); +//// } +//// } +//// fm.setTopics(newTopics); +//// } +//// +//// // create topic to content map +//// HashMap> contentByTopic = new HashMap<>(); +//// for (ParsedMarkdown parsedMarkdown : markdownInfos) { +//// for (String topic : parsedMarkdown.getFrontmatter().getTopics()) { +//// contentByTopic.computeIfAbsent(topic, t -> new ArrayList<>()).add(parsedMarkdown); +//// } +//// } +//// +//// ListIterator lit = markdownInfos.listIterator(); +//// while (lit.hasNext()) { +//// MarkdownInfo mif = lit.next(); +//// if (mif.hasAggregations()) { +//// lit.remove(); +//// mif = new CompositeMarkdownInfo().add(mif); +//// lit.add(mif); +//// } +//// } +//// +//// // combine aggregate targets +//// for (ParsedMarkdown parsedMarkdown : markdownInfos) { +//// List aggregations = parsedMarkdown.getFrontmatter().getAggregations(); +//// if (aggregations.size()>0) { +//// for (Pattern aggregation : aggregations) { +//// +//// } +//// } +//// } +//// +//// // Assign glob topics +//// +//// // Assign content aggregates +//// System.out.println("topics: " + topicSets); +//// +//// aggregated.addAll(markdownInfos); +// return aggregated; } diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ParsedFrontMatter.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ParsedFrontMatter.java index 1b88328c3..e87173523 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ParsedFrontMatter.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ParsedFrontMatter.java @@ -52,11 +52,23 @@ public class ParsedFrontMatter implements FrontMatterInfo { for (String topic : topics) { Collections.addAll(topicSet, topic.split(", *")); } - topicSet.addAll(topics); +// topicSet.addAll(topics); } return topicSet; } + @Override + public List getIncluded() { + List included = data.get(FrontMatterInfo.INCLUDED); + List includedList = new ArrayList<>(); + if (included!=null) { + for (String s : included) { + Collections.addAll(includedList, s.split(", *")); + } + } + return includedList; + } + @Override public List getAggregations() { if (!data.containsKey(FrontMatterInfo.AGGREGATE)) { @@ -104,10 +116,30 @@ public class ParsedFrontMatter implements FrontMatterInfo { return new ParsedFrontMatter(newmap); } + public ParsedFrontMatter withIncluded(List included) { + HashMap> newmap = new HashMap<>(); + newmap.putAll(this.data); + newmap.put(FrontMatterInfo.INCLUDED,included); + return new ParsedFrontMatter(newmap); + } + @Override public String toString() { return "ParsedFrontMatter{" + "data=" + data + '}'; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ParsedFrontMatter that = (ParsedFrontMatter) o; + return Objects.equals(data, that.data); + } + + @Override + public int hashCode() { + return Objects.hash(data); + } } diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ParsedMarkdown.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ParsedMarkdown.java index fdbcc4b4a..3a6657956 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ParsedMarkdown.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ParsedMarkdown.java @@ -11,10 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; +import java.util.*; /** * TODO: Make this a value type @@ -93,9 +90,27 @@ public class ParsedMarkdown implements MarkdownInfo, HasDiagnostics { return new ParsedMarkdown(frontMatter.withTopics(assigning), this.content); } + public MarkdownInfo withIncluded(List included) { + return new ParsedMarkdown(frontMatter.withIncluded(included), this.content); + } + @Override public String toString() { return "ParsedMarkdown/" + frontMatter.toString(); } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ParsedMarkdown that = (ParsedMarkdown) o; + return Objects.equals(frontMatter, that.frontMatter) && + Objects.equals(content, that.content); + } + + @Override + public int hashCode() { + return Objects.hash(frontMatter, content); + } } diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/FrontMatterInfo.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/FrontMatterInfo.java index aab8f42ad..a3e5c0d6c 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/FrontMatterInfo.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/FrontMatterInfo.java @@ -16,7 +16,10 @@ public interface FrontMatterInfo { String TOPICS = "topics"; String WEIGHT = "weight"; String TITLE = "title"; - Set FrontMatterKeyWords = Set.of(SCOPES, AGGREGATE,TOPICS,WEIGHT,TITLE); + String INCLUDED = "included"; + + Set FrontMatterKeyWords = + Set.of(SCOPES, AGGREGATE,TOPICS,WEIGHT, TITLE,INCLUDED); /** @@ -50,6 +53,18 @@ public interface FrontMatterInfo { */ Set getTopics(); + /** + *

If content is included in an item from another topic, then the + * topic name with which the additional content was added is in the + * inclueded list of topics.

+ * + *

This is distinct from {@link #getTopics()}, which is not modified + * by the included topic names.

+ * + * @return A list of included topics. + */ + List getIncluded(); + /** *

* Aggregation patterns coalesce all the topics that they match into a seamless logical diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/MarkdownInfo.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/MarkdownInfo.java index 0f171a4ee..0d741198e 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/MarkdownInfo.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/MarkdownInfo.java @@ -1,13 +1,15 @@ package io.nosqlbench.nb.api.markdown.types; +import io.nosqlbench.nb.api.markdown.aggregator.CompositeMarkdownInfo; import io.nosqlbench.nb.api.markdown.types.FrontMatterInfo; +import org.jetbrains.annotations.NotNull; import java.nio.file.Path; import java.util.List; import java.util.regex.Pattern; import java.util.stream.Collectors; -public interface MarkdownInfo { +public interface MarkdownInfo extends Comparable { Path getPath(); @@ -18,27 +20,53 @@ public interface MarkdownInfo { boolean hasAggregations(); default boolean hasTopicGlobs() { - return getTopicGlobs().size()>0; + return getTopicGlobs().size() > 0; } + default List getTopicGlobs() { - return getFrontmatter().getTopics().stream() - .filter(t -> t.startsWith("^") || t.endsWith("$") || t.contains(".*") || t.contains(".+")) - .map(Pattern::compile) - .collect(Collectors.toList()); + List pattern = getFrontmatter().getTopics().stream() + .filter(t -> t.startsWith("^") || t.endsWith("$") || t.contains(".*") || t.contains(".+")) + .map(Pattern::compile) + .collect(Collectors.toList()); + return pattern; } default List getTopics() { return getFrontmatter().getTopics().stream() - .filter(t -> !t.startsWith("^") && !t.endsWith("$") && !t.contains(".*") && !t.contains(".+")) - .collect(Collectors.toList()); + .filter(t -> !t.startsWith("^") && !t.endsWith("$") && !t.contains(".*") && !t.contains(".+")) + .collect(Collectors.toList()); } - default boolean hasAggregators() { - return getFrontmatter().getAggregations().size()>0; + default List getIncluded() { + return getFrontmatter().getIncluded(); } + + + default boolean hasAggregators() { + return getFrontmatter().getAggregations().size() > 0; + } + default List getAggregators() { return getFrontmatter().getAggregations(); } MarkdownInfo withTopics(List assigning); + + default int compareTo(@NotNull MarkdownInfo o) { + int diff = getFrontmatter().getWeight() - o.getFrontmatter().getWeight(); + if (diff != 0) return diff; + diff = getFrontmatter().getTitle().compareTo(o.getFrontmatter().getTitle()); + if (diff!=0) return diff; + diff = getBody().compareTo(o.getBody()); + return diff; + } + + default boolean matchesTopicPattern(Pattern pattern) { + return getTopics().stream().anyMatch(t -> pattern.matcher(t).matches()); + } + + MarkdownInfo withIncluded(List included); + + + } diff --git a/nb-api/src/main/resources/docs-for-testing-only/srcmain-topics-a/srcmain-entry1-1.md b/nb-api/src/main/resources/docs-for-testing-logical/srcmain-topics-a/srcmain-entry1-1.md similarity index 100% rename from nb-api/src/main/resources/docs-for-testing-only/srcmain-topics-a/srcmain-entry1-1.md rename to nb-api/src/main/resources/docs-for-testing-logical/srcmain-topics-a/srcmain-entry1-1.md diff --git a/nb-api/src/main/resources/docs-for-testing-only/srcmain-topics-b/srcmain-topic2-1/srcmain-entry-2-1-L.md b/nb-api/src/main/resources/docs-for-testing-logical/srcmain-topics-b/srcmain-topic2-1/srcmain-entry-2-1-L.md similarity index 100% rename from nb-api/src/main/resources/docs-for-testing-only/srcmain-topics-b/srcmain-topic2-1/srcmain-entry-2-1-L.md rename to nb-api/src/main/resources/docs-for-testing-logical/srcmain-topics-b/srcmain-topic2-1/srcmain-entry-2-1-L.md diff --git a/nb-api/src/main/resources/docs-for-testing-raw/srcmain-topics-a/srcmain-entry1-1.md b/nb-api/src/main/resources/docs-for-testing-raw/srcmain-topics-a/srcmain-entry1-1.md new file mode 100644 index 000000000..7d00bb83e --- /dev/null +++ b/nb-api/src/main/resources/docs-for-testing-raw/srcmain-topics-a/srcmain-entry1-1.md @@ -0,0 +1,10 @@ +--- +title: srcmain-Entry 1-1 +weight: 37 +topics: entries/entry2-1, related-topic-for-entry1-1 +aggregate: topic +--- + +# Title Heading for srcmain-Entry 1-1 + + diff --git a/nb-api/src/main/resources/docs-for-testing-raw/srcmain-topics-b/srcmain-topic2-1/srcmain-entry-2-1-L.md b/nb-api/src/main/resources/docs-for-testing-raw/srcmain-topics-b/srcmain-topic2-1/srcmain-entry-2-1-L.md new file mode 100644 index 000000000..e5d272e62 --- /dev/null +++ b/nb-api/src/main/resources/docs-for-testing-raw/srcmain-topics-b/srcmain-topic2-1/srcmain-entry-2-1-L.md @@ -0,0 +1,8 @@ +--- +title: srcmain-Entry 2-1-L +weight: 39 +--- + +# Title Heading for srcmain-Entry 2-1-L + + diff --git a/nb-api/src/test/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocsTest.java b/nb-api/src/test/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocsTest.java index f9136d823..5e79041f0 100644 --- a/nb-api/src/test/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocsTest.java +++ b/nb-api/src/test/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocsTest.java @@ -1,19 +1,87 @@ package io.nosqlbench.nb.api.markdown.aggregator; +import io.nosqlbench.nb.api.content.PathContent; import io.nosqlbench.nb.api.markdown.types.MarkdownInfo; import org.junit.Test; -import java.util.List; +import java.net.URL; +import java.nio.file.*; +import java.util.*; +import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.from; public class MarkdownDocsTest { @Test public void testLoadMarkdown() { - List all = MarkdownDocs.findAll(); -// assertThat(all).hasSizeGreaterThan(0); + List processed = MarkdownDocs.findAll(); + List expected = fromRaw("docs-for-testing-logical"); + + Map processedPaths = processed.stream().collect(Collectors.toMap(MarkdownInfo::getPath, v -> v)); + Map expectedPaths = expected.stream().collect(Collectors.toMap(MarkdownInfo::getPath, v -> v)); + + for (Path path : expectedPaths.keySet()) { + System.out.println("expected path:" + path.toString()); + } + + Set missingPaths = new HashSet<>(); + for (Path path : expectedPaths.keySet()) { + if (!processedPaths.containsKey(path)) { + missingPaths.add(path); + } + } + + Set extraPaths = new HashSet<>(); + for (Path path : processedPaths.keySet()) { + if (!expectedPaths.containsKey(path)) { + extraPaths.add(path); + } + } + + for (MarkdownInfo markdownInfo : processed) { + Path path = markdownInfo.getPath(); + } + + assertThat(missingPaths).isEmpty(); + assertThat(extraPaths).isEmpty(); } + private List fromRaw(String parentPath) { + List fromraw = new ArrayList<>(); + List postpaths = getSubPaths("docs-for-testing-logical"); + for (Path postpath : postpaths) { + PathContent content = new PathContent(postpath); + ParsedMarkdown parsedMarkdown = new ParsedMarkdown(content); + fromraw.add(parsedMarkdown); + } + Collections.sort(fromraw); + return fromraw; + } + + private static List getSubPaths(String resourcePath) { + List subpaths = new ArrayList<>(); + + try { + Enumeration resources = + MarkdownDocsTest.class.getClassLoader().getResources(resourcePath); + + while (resources.hasMoreElements()) { + URL url = resources.nextElement(); + System.out.println("url="+url.toExternalForm()); + Path path = Paths.get(url.toURI()); + Files.walk(path, FileVisitOption.FOLLOW_LINKS) + .filter(p -> !Files.isDirectory(p, LinkOption.NOFOLLOW_LINKS)) + .forEach(subpaths::add); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return subpaths; + + } + + }