From 96e88ee68d58d87d6f44179bda324327bd9ca5bf Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Fri, 1 May 2020 13:56:49 -0500 Subject: [PATCH] incremental progress on docs system --- .../nosqlbench/nb/api/content/NBIOWalker.java | 1 + .../{aggregator => }/DocsForTestingOnly.java | 2 +- .../aggregator/CompositeMarkdownInfo.java | 43 ++++++++ ...stSplitterWhyDoesJavaNotDoThisAlready.java | 21 ++++ .../api/markdown/aggregator/MarkdownDocs.java | 97 ++++++++++++++++-- .../api/markdown/aggregator/MarkdownInfo.java | 11 --- .../aggregator/MutableFrontMatter.java | 73 -------------- .../aggregator/MutableMarkdownInfo.java | 41 -------- .../aggregator/ParsedFrontMatter.java | 98 +++++++++++++++++++ .../markdown/aggregator/ParsedMarkdown.java | 85 ++++++++++++++++ .../markdown/exporter/MarkdownExporter.java | 4 +- .../nb/api/markdown/types/Diagnostics.java | 20 ++++ .../{aggregator => types}/DocScope.java | 8 +- .../FrontMatterInfo.java} | 10 +- .../nb/api/markdown/types/HasDiagnostics.java | 8 ++ .../nb/api/markdown/types/MarkdownInfo.java | 42 ++++++++ .../srcmain-entry1-1.md} | 4 +- .../srcmain-topic2-1/srcmain-entry-2-1-L.md | 8 ++ .../markdown/aggregator/MarkdownDocsTest.java | 2 +- .../srctest-topics-a/srctest-entry1-1.md | 12 +++ .../intesting-entry-2-1-L.md | 8 ++ .../topics-a/entry1-1.md | 10 -- 22 files changed, 449 insertions(+), 159 deletions(-) rename nb-api/src/main/java/io/nosqlbench/nb/api/markdown/{aggregator => }/DocsForTestingOnly.java (88%) create mode 100644 nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/CompositeMarkdownInfo.java create mode 100644 nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ListSplitterWhyDoesJavaNotDoThisAlready.java delete mode 100644 nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownInfo.java delete mode 100644 nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableFrontMatter.java delete mode 100644 nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableMarkdownInfo.java create mode 100644 nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ParsedFrontMatter.java create mode 100644 nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ParsedMarkdown.java create mode 100644 nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/Diagnostics.java rename nb-api/src/main/java/io/nosqlbench/nb/api/markdown/{aggregator => types}/DocScope.java (94%) rename nb-api/src/main/java/io/nosqlbench/nb/api/markdown/{aggregator/FrontMatter.java => types/FrontMatterInfo.java} (91%) create mode 100644 nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/HasDiagnostics.java create mode 100644 nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/MarkdownInfo.java rename nb-api/src/main/resources/docs-for-testing-only/{topics-a/entry1-1.md => srcmain-topics-a/srcmain-entry1-1.md} (59%) create mode 100644 nb-api/src/main/resources/docs-for-testing-only/srcmain-topics-b/srcmain-topic2-1/srcmain-entry-2-1-L.md create mode 100644 nb-api/src/test/resources/docs-for-testing-only/srctest-topics-a/srctest-entry1-1.md create mode 100644 nb-api/src/test/resources/docs-for-testing-only/srctest-topics-b/intesting-topic2-1/intesting-entry-2-1-L.md delete mode 100644 nb-api/src/test/resources/docs-for-testing-only/topics-a/entry1-1.md diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/content/NBIOWalker.java b/nb-api/src/main/java/io/nosqlbench/nb/api/content/NBIOWalker.java index 86bd206db..938b6e950 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/content/NBIOWalker.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/content/NBIOWalker.java @@ -14,6 +14,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; public class NBIOWalker { + private final static Logger logger = LogManager.getLogger(NBIOWalker.class); public static void walk(Path p, PathVisitor v) { diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/DocsForTestingOnly.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/DocsForTestingOnly.java similarity index 88% rename from nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/DocsForTestingOnly.java rename to nb-api/src/main/java/io/nosqlbench/nb/api/markdown/DocsForTestingOnly.java index 2f3eb0e2d..9b78d13d2 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/DocsForTestingOnly.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/DocsForTestingOnly.java @@ -1,4 +1,4 @@ -package io.nosqlbench.nb.api.markdown.aggregator; +package io.nosqlbench.nb.api.markdown; import io.nosqlbench.nb.annotations.Service; import io.nosqlbench.nb.api.markdown.providers.DocsRootDirectory; diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/CompositeMarkdownInfo.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/CompositeMarkdownInfo.java new file mode 100644 index 000000000..996947f89 --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/CompositeMarkdownInfo.java @@ -0,0 +1,43 @@ +package io.nosqlbench.nb.api.markdown.aggregator; + +import io.nosqlbench.nb.api.markdown.types.FrontMatterInfo; +import io.nosqlbench.nb.api.markdown.types.MarkdownInfo; + +import java.nio.file.Path; +import java.util.LinkedList; +import java.util.List; + +public class CompositeMarkdownInfo implements MarkdownInfo { + private List elements = new LinkedList<>(); + + @Override + public Path getPath() { + return elements.get(0).getPath(); + } + + @Override + public String getBody() { + StringBuilder sb = new StringBuilder(); + for (MarkdownInfo element : elements) { + sb.append(element.getBody()); + } + return sb.toString(); + } + + @Override + public FrontMatterInfo getFrontmatter() { + // calculate included topics + return null; + } + + @Override + public boolean hasAggregations() { + // was true, but now it is false after compositing + return false; + } + + public CompositeMarkdownInfo add(T element) { + elements.add(element); + return this; + } +} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ListSplitterWhyDoesJavaNotDoThisAlready.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ListSplitterWhyDoesJavaNotDoThisAlready.java new file mode 100644 index 000000000..862c87b11 --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ListSplitterWhyDoesJavaNotDoThisAlready.java @@ -0,0 +1,21 @@ +package io.nosqlbench.nb.api.markdown.aggregator; + +import java.util.ArrayList; +import java.util.List; +import java.util.ListIterator; +import java.util.function.Predicate; + +public class ListSplitterWhyDoesJavaNotDoThisAlready { + public static List partition(List source, Predicate filterout) { + ArrayList filtered = new ArrayList<>(); + ListIterator it = source.listIterator(); + while (it.hasNext()) { + T element = it.next(); + if (filterout.test(element)) { + it.remove(); + filtered.add(element); + } + } + return filtered; + } +} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocs.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocs.java index 00aec6bc9..a92e89d94 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocs.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocs.java @@ -1,18 +1,22 @@ package io.nosqlbench.nb.api.markdown.aggregator; -import com.vladsch.flexmark.ext.yaml.front.matter.AbstractYamlFrontMatterVisitor; -import com.vladsch.flexmark.ext.yaml.front.matter.YamlFrontMatterExtension; -import com.vladsch.flexmark.html2md.converter.FlexmarkHtmlConverter; -import com.vladsch.flexmark.util.ast.Document; import io.nosqlbench.nb.api.content.Content; -import io.nosqlbench.nb.api.markdown.FlexParser; import io.nosqlbench.nb.api.markdown.providers.RawMarkdownSources; +import io.nosqlbench.nb.api.markdown.types.Diagnostics; +import io.nosqlbench.nb.api.markdown.types.DocScope; +import io.nosqlbench.nb.api.markdown.types.FrontMatterInfo; +import io.nosqlbench.nb.api.markdown.types.MarkdownInfo; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.*; +import java.util.regex.Pattern; import java.util.stream.Collectors; public class MarkdownDocs { + private final static Logger logger = LogManager.getLogger(MarkdownDocs.class); + public static List find(DocScope... scopes) { return find(".*", scopes); } @@ -29,18 +33,85 @@ public class MarkdownDocs { // Find all topics and aggregators List aggregators = new ArrayList<>(); - List markdownInfos = markdownContent + List markdownInfos = markdownContent .stream() - .map(MutableMarkdownInfo::new) + .map(ParsedMarkdown::new) .collect(Collectors.toList()); - List> collect = + List diagBuffer = new ArrayList<>(); + markdownInfos.forEach(i -> Diagnostics.getDiagnostics(i,diagBuffer)); + diagBuffer.forEach(logger::warn); + + + List> topicSets = markdownInfos.stream().map(m -> m.getFrontmatter().getTopics()).collect(Collectors.toList()); + // find non-glob topics + List nonGlobTopics = markdownInfos.stream() + .map(MarkdownInfo::getFrontmatter) + .map(FrontMatterInfo::getTopics).flatMap(Collection::stream) + .filter(s -> !isPattern(s)) + .collect(Collectors.toList()); + + List markdownWithTopicGlobs = + ListSplitterWhyDoesJavaNotDoThisAlready.partition(markdownInfos, MarkdownInfo::hasTopicGlobs); + Collections. + + for (MarkdownInfo markdownWithTopicGlob : markdownWithTopicGlobs) { + + } + // Assign glob topics to non-glob topics that match + for (MarkdownInfo parsedMarkdown : markdownInfos) { + FrontMatterInfo fm = parsedMarkdown.getFrontmatter(); + Set topics = fm.getTopics(); + Set newTopics = new HashSet<>(); + for (String topic : topics) { + if (isPattern(topic)) { + Pattern p = Pattern.compile(topic); + for (String nonGlobTopic : nonGlobTopics) { + if (p.matcher(nonGlobTopic).matches()) { + newTopics.add(topic); + } + } + } else { + newTopics.add(topic); + } + } + fm.setTopics(newTopics); + } + + // create topic to content map + HashMap> contentByTopic = new HashMap<>(); + for (ParsedMarkdown parsedMarkdown : markdownInfos) { + for (String topic : parsedMarkdown.getFrontmatter().getTopics()) { + contentByTopic.computeIfAbsent(topic, t -> new ArrayList<>()).add(parsedMarkdown); + } + } + + ListIterator lit = markdownInfos.listIterator(); + while (lit.hasNext()) { + MarkdownInfo mif = lit.next(); + if (mif.hasAggregations()) { + lit.remove(); + mif = new CompositeMarkdownInfo().add(mif); + lit.add(mif); + } + } + + // combine aggregate targets + for (ParsedMarkdown parsedMarkdown : markdownInfos) { + List aggregations = parsedMarkdown.getFrontmatter().getAggregations(); + if (aggregations.size()>0) { + for (Pattern aggregation : aggregations) { + + } + } + } + // Assign glob topics // Assign content aggregates - System.out.println("topics: " + collect); + System.out.println("topics: " + topicSets); aggregated.addAll(markdownInfos); return aggregated; @@ -48,4 +119,12 @@ public class MarkdownDocs { } + private static boolean isPattern(String srcTopic) { + return + srcTopic.startsWith("^") + || srcTopic.contains(".*") + || srcTopic.contains(".+") + || srcTopic.endsWith("$"); + } + } diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownInfo.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownInfo.java deleted file mode 100644 index a33b5ec62..000000000 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownInfo.java +++ /dev/null @@ -1,11 +0,0 @@ -package io.nosqlbench.nb.api.markdown.aggregator; - -import java.nio.file.Path; - -public interface MarkdownInfo { - - Path getPath(); - String getBody(); - FrontMatter getFrontmatter(); - -} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableFrontMatter.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableFrontMatter.java deleted file mode 100644 index 999d093a2..000000000 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableFrontMatter.java +++ /dev/null @@ -1,73 +0,0 @@ -package io.nosqlbench.nb.api.markdown.aggregator; - -import java.security.InvalidParameterException; -import java.util.*; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -public class MutableFrontMatter implements FrontMatter { - private final Map> data; - - public MutableFrontMatter(Map> data) { - this.data = data; - } - - @Override - public String getTitle() { - List titles = data.get(FrontMatter.TITLE); - if (titles==null) { - return ""; - } - if (titles.size()!=1) { - throw new InvalidParameterException(FrontMatter.TITLE + " can only contain a single value."); - } - return titles.get(0); - } - - @Override - public int getWeight() { - List weights = data.get(FrontMatter.WEIGHT); - if (weights==null) { - return 0; - } - if (weights.size()!=1) { - throw new InvalidParameterException(FrontMatter.WEIGHT + " can only contain a single value."); - } - return Integer.parseInt(weights.get(0)); - } - - @Override - public Set getTopics() { - - List topics = data.get(FrontMatter.TOPICS); - List topic = data.get(FrontMatter.TOPIC); - - if (topics==null && topic==null) { - return Set.of(); - } - Set topicSet = new HashSet<>(); - if (topics!=null) { - topicSet.addAll(topics); - } - if (topic!=null) { - topicSet.addAll(topic); - } - return topicSet; - } - - @Override - public List getAggregations() { - if (!data.containsKey(FrontMatter.AGGREGATIONS)) { - return List.of(); - } - return data.get(FrontMatter.AGGREGATIONS).stream().map(Pattern::compile).collect(Collectors.toList()); - } - - @Override - public Set getDocScopes() { - if (!data.containsKey(FrontMatter.SCOPE)) { - return Set.of(DocScope.NONE); - } - return data.get(FrontMatter.SCOPE).stream().map(DocScope::valueOf).collect(Collectors.toSet()); - } -} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableMarkdownInfo.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableMarkdownInfo.java deleted file mode 100644 index fc323b716..000000000 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableMarkdownInfo.java +++ /dev/null @@ -1,41 +0,0 @@ -package io.nosqlbench.nb.api.markdown.aggregator; - -import com.vladsch.flexmark.ext.yaml.front.matter.AbstractYamlFrontMatterVisitor; -import com.vladsch.flexmark.util.ast.Document; -import io.nosqlbench.nb.api.content.Content; -import io.nosqlbench.nb.api.markdown.FlexParser; - -import java.nio.file.Path; -import java.util.List; -import java.util.Map; - -public class MutableMarkdownInfo implements MarkdownInfo { - - private final FrontMatter frontMatter; - private final Content content; - - public MutableMarkdownInfo(Content content) { - String rawMarkdown = content.asString(); - AbstractYamlFrontMatterVisitor v = new AbstractYamlFrontMatterVisitor(); - Document parsed = FlexParser.parser.parse(rawMarkdown); - v.visit(parsed); - Map> data = v.getData(); - frontMatter = new MutableFrontMatter(data); - this.content = content; - } - - @Override - public Path getPath() { - return content.asPath(); - } - - @Override - public String getBody() { - return null; - } - - @Override - public FrontMatter getFrontmatter() { - return frontMatter; - } -} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ParsedFrontMatter.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ParsedFrontMatter.java new file mode 100644 index 000000000..cc71ec1f6 --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ParsedFrontMatter.java @@ -0,0 +1,98 @@ +package io.nosqlbench.nb.api.markdown.aggregator; + +import io.nosqlbench.nb.api.markdown.types.DocScope; +import io.nosqlbench.nb.api.markdown.types.FrontMatterInfo; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.security.InvalidParameterException; +import java.util.*; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class ParsedFrontMatter implements FrontMatterInfo { + + private final static Logger logger = LogManager.getLogger(ParsedFrontMatter.class); + + private final Map> data; + + public ParsedFrontMatter(Map> data) { + this.data = data; + } + + @Override + public String getTitle() { + List titles = data.get(FrontMatterInfo.TITLE); + if (titles==null) { + return ""; + } + if (titles.size()!=1) { + throw new InvalidParameterException(FrontMatterInfo.TITLE + " can only contain a single value."); + } + return titles.get(0); + } + + @Override + public int getWeight() { + List weights = data.get(FrontMatterInfo.WEIGHT); + if (weights==null) { + return 0; + } + if (weights.size()!=1) { + throw new InvalidParameterException(FrontMatterInfo.WEIGHT + " can only contain a single value."); + } + return Integer.parseInt(weights.get(0)); + } + + @Override + public Set getTopics() { + List topics = data.get(FrontMatterInfo.TOPICS); + Set topicSet = new HashSet<>(); + if (topics!=null) { + for (String topic : topics) { + Collections.addAll(topicSet, topic.split(", *")); + } + topicSet.addAll(topics); + } + return topicSet; + } + + @Override + public List getAggregations() { + if (!data.containsKey(FrontMatterInfo.AGGREGATE)) { + return List.of(); + } + List patterns = new ArrayList<>(); + for (String aggName : data.get(FrontMatterInfo.AGGREGATE)) { + Collections.addAll(patterns,aggName.split(", *")); + } + return patterns.stream().map(Pattern::compile).collect(Collectors.toList()); + } + + @Override + public Set getDocScopes() { + if (!data.containsKey(FrontMatterInfo.SCOPES)) { + return Set.of(DocScope.NONE); + } + List scopeNames = new ArrayList<>(); + for (String scopeName : data.get(FrontMatterInfo.SCOPES)) { + Collections.addAll(scopeNames,scopeName.split(", *")); + } + return scopeNames.stream().map(DocScope::valueOf).collect(Collectors.toSet()); + } + + public List getDiagnostics() { + List warnings = new ArrayList<>(); + for (String propname : data.keySet()) { + if (!FrontMatterInfo.FrontMatterKeyWords.contains(propname)) { + warnings.add("unrecognized frontm atter property " + propname); + } + } + return warnings; + } + + + public void setTopics(Set newTopics) { + this.data.put(FrontMatterInfo.TOPICS,newTopics); + } +} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ParsedMarkdown.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ParsedMarkdown.java new file mode 100644 index 000000000..c63c91c6a --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ParsedMarkdown.java @@ -0,0 +1,85 @@ +package io.nosqlbench.nb.api.markdown.aggregator; + +import com.vladsch.flexmark.ext.yaml.front.matter.AbstractYamlFrontMatterVisitor; +import com.vladsch.flexmark.util.ast.Document; +import io.nosqlbench.nb.api.content.Content; +import io.nosqlbench.nb.api.markdown.FlexParser; +import io.nosqlbench.nb.api.markdown.types.FrontMatterInfo; +import io.nosqlbench.nb.api.markdown.types.HasDiagnostics; +import io.nosqlbench.nb.api.markdown.types.MarkdownInfo; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * TODO: Make this a value type + */ +public class ParsedMarkdown implements MarkdownInfo, HasDiagnostics { + private final static Logger logger = LogManager.getLogger(MarkdownDocs.class); + + private final ParsedFrontMatter frontMatter; + private final Content content; + + public ParsedMarkdown(Content content) { + String rawMarkdown = content.asString(); + AbstractYamlFrontMatterVisitor v = new AbstractYamlFrontMatterVisitor(); + Document parsed = FlexParser.parser.parse(rawMarkdown); + v.visit(parsed); + Map> data = v.getData(); + frontMatter = new ParsedFrontMatter(data); + this.content = content; + logger.debug("created " + this.toString()); + } + + @Override + public Path getPath() { + return content.asPath(); + } + + @Override + public String getBody() { + return null; + } + + @Override + public FrontMatterInfo getFrontmatter() { + return frontMatter; + } + + /** + * Get a list of diagnostic warnings that might help users know of issues in their + * markdown content before publication. + * @param buffer A buffer object, for accumulating many lines of detail, if necessary. + * @return The buffer, with possible additions + */ + @Override + public List getDiagnostics(List buffer) { + List diagnostics = frontMatter.getDiagnostics(); + if (diagnostics.size()==0) { + return List.of(); + } + String[] diags = diagnostics.stream().map(s -> " " + s).toArray(String[]::new); + buffer.add("found " + diagnostics.size() + " diagnostics for " + getPath().toString()); + buffer.addAll(Arrays.asList(diags)); + return buffer; + } + + /** + * The buffer-less version of {@link #getDiagnostics(List)} + * @return a list of diagnostics lines, zero if there are none + */ + @Override + public List getDiagnostics() { + return getDiagnostics(new ArrayList<>()); + } + + @Override + public boolean hasAggregations() { + return getFrontmatter().getAggregations().size()>0; + } +} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/exporter/MarkdownExporter.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/exporter/MarkdownExporter.java index b1ef00240..828883c91 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/exporter/MarkdownExporter.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/exporter/MarkdownExporter.java @@ -1,8 +1,8 @@ package io.nosqlbench.nb.api.markdown.exporter; -import io.nosqlbench.nb.api.markdown.aggregator.DocScope; +import io.nosqlbench.nb.api.markdown.types.DocScope; import io.nosqlbench.nb.api.markdown.aggregator.MarkdownDocs; -import io.nosqlbench.nb.api.markdown.aggregator.MarkdownInfo; +import io.nosqlbench.nb.api.markdown.types.MarkdownInfo; import joptsimple.*; import java.nio.file.Path; diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/Diagnostics.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/Diagnostics.java new file mode 100644 index 000000000..71586442f --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/Diagnostics.java @@ -0,0 +1,20 @@ +package io.nosqlbench.nb.api.markdown.types; + +import io.nosqlbench.nb.api.markdown.types.HasDiagnostics; + +import java.util.List; + +public class Diagnostics { + public static List getDiagnostics(Object o) { + if (o instanceof HasDiagnostics) { + return ((HasDiagnostics)o).getDiagnostics(); + } + return List.of(); + } + public static List getDiagnostics(Object o, List buffer) { + if (o instanceof HasDiagnostics) { + return ((HasDiagnostics)o).getDiagnostics(buffer); + } + return buffer; + } +} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/DocScope.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/DocScope.java similarity index 94% rename from nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/DocScope.java rename to nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/DocScope.java index b788bd207..c963bf8ed 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/DocScope.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/DocScope.java @@ -1,4 +1,4 @@ -package io.nosqlbench.nb.api.markdown.aggregator; +package io.nosqlbench.nb.api.markdown.types; /** * DocScope determines which display mode a topic is meant to be displayed in. @@ -13,19 +13,19 @@ public enum DocScope { * The command line doc scope includes any markdown which should be shown to the user * when they are searching for or viewing documentation on a command line. */ - CommandLine(false), + cli(false), /** * The static web doc scope includes any markdown which should be shown to the user * when they are viewing documentation on an externally hosted site in static form. */ - StaticWeb(false), + web(false), /** * The dynamic web doc scope includes any markdown which should be made available to * users when they are interacting with a web application. */ - DynamicWeb(false), + app(false), /** * ANY is a descriptive doc scope which is meant to be used as a filter within API calls diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/FrontMatter.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/FrontMatterInfo.java similarity index 91% rename from nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/FrontMatter.java rename to nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/FrontMatterInfo.java index b1599d754..aab8f42ad 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/FrontMatter.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/FrontMatterInfo.java @@ -1,4 +1,4 @@ -package io.nosqlbench.nb.api.markdown.aggregator; +package io.nosqlbench.nb.api.markdown.types; import java.util.List; import java.util.Set; @@ -9,14 +9,14 @@ import java.util.regex.Pattern; * If the markdown source file does not contain the metadata requested, then reasonable non-null * defaults must be provided. */ -public interface FrontMatter { +public interface FrontMatterInfo { - String SCOPE = "scope"; - String AGGREGATIONS = "aggregations"; + String SCOPES = "scopes"; + String AGGREGATE = "aggregate"; String TOPICS = "topics"; - String TOPIC = "topic"; String WEIGHT = "weight"; String TITLE = "title"; + Set FrontMatterKeyWords = Set.of(SCOPES, AGGREGATE,TOPICS,WEIGHT,TITLE); /** diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/HasDiagnostics.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/HasDiagnostics.java new file mode 100644 index 000000000..a60ab48a8 --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/HasDiagnostics.java @@ -0,0 +1,8 @@ +package io.nosqlbench.nb.api.markdown.types; + +import java.util.List; + +public interface HasDiagnostics { + List getDiagnostics(List buffer); + List getDiagnostics(); +} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/MarkdownInfo.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/MarkdownInfo.java new file mode 100644 index 000000000..0991f9ba7 --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/MarkdownInfo.java @@ -0,0 +1,42 @@ +package io.nosqlbench.nb.api.markdown.types; + +import io.nosqlbench.nb.api.markdown.types.FrontMatterInfo; + +import java.nio.file.Path; +import java.util.List; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public interface MarkdownInfo { + + Path getPath(); + + String getBody(); + + FrontMatterInfo getFrontmatter(); + + boolean hasAggregations(); + + default boolean hasTopicGlobs() { + return getTopicGlobs().size()>0; + } + default List getTopicGlobs() { + return getFrontmatter().getTopics().stream() + .filter(t -> t.startsWith("^") || t.endsWith("$") || t.contains(".*") || t.contains(".+")) + .map(Pattern::compile) + .collect(Collectors.toList()); + } + + default List getTopics() { + return getFrontmatter().getTopics().stream() + .filter(t -> !t.startsWith("^") && !t.endsWith("$") && !t.contains(".*") && !t.contains(".+")) + .collect(Collectors.toList()); + } + + default boolean hasAggregators() { + return getFrontmatter().getAggregations().size()>0; + } + default List getAggregators() { + return getFrontmatter().getAggregations(); + } +} diff --git a/nb-api/src/main/resources/docs-for-testing-only/topics-a/entry1-1.md b/nb-api/src/main/resources/docs-for-testing-only/srcmain-topics-a/srcmain-entry1-1.md similarity index 59% rename from nb-api/src/main/resources/docs-for-testing-only/topics-a/entry1-1.md rename to nb-api/src/main/resources/docs-for-testing-only/srcmain-topics-a/srcmain-entry1-1.md index 6555ee44e..7d00bb83e 100644 --- a/nb-api/src/main/resources/docs-for-testing-only/topics-a/entry1-1.md +++ b/nb-api/src/main/resources/docs-for-testing-only/srcmain-topics-a/srcmain-entry1-1.md @@ -1,10 +1,10 @@ --- -title: Entry 1-1 +title: srcmain-Entry 1-1 weight: 37 topics: entries/entry2-1, related-topic-for-entry1-1 aggregate: topic --- -# Title Heading for Entry 1-1 +# Title Heading for srcmain-Entry 1-1 diff --git a/nb-api/src/main/resources/docs-for-testing-only/srcmain-topics-b/srcmain-topic2-1/srcmain-entry-2-1-L.md b/nb-api/src/main/resources/docs-for-testing-only/srcmain-topics-b/srcmain-topic2-1/srcmain-entry-2-1-L.md new file mode 100644 index 000000000..e5d272e62 --- /dev/null +++ b/nb-api/src/main/resources/docs-for-testing-only/srcmain-topics-b/srcmain-topic2-1/srcmain-entry-2-1-L.md @@ -0,0 +1,8 @@ +--- +title: srcmain-Entry 2-1-L +weight: 39 +--- + +# Title Heading for srcmain-Entry 2-1-L + + diff --git a/nb-api/src/test/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocsTest.java b/nb-api/src/test/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocsTest.java index 4f7de96e6..992b45eb3 100644 --- a/nb-api/src/test/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocsTest.java +++ b/nb-api/src/test/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocsTest.java @@ -1,6 +1,6 @@ package io.nosqlbench.nb.api.markdown.aggregator; -import org.junit.Ignore; +import io.nosqlbench.nb.api.markdown.types.MarkdownInfo; import org.junit.Test; import java.util.List; diff --git a/nb-api/src/test/resources/docs-for-testing-only/srctest-topics-a/srctest-entry1-1.md b/nb-api/src/test/resources/docs-for-testing-only/srctest-topics-a/srctest-entry1-1.md new file mode 100644 index 000000000..bf896cd0a --- /dev/null +++ b/nb-api/src/test/resources/docs-for-testing-only/srctest-topics-a/srctest-entry1-1.md @@ -0,0 +1,12 @@ +--- +title: srctest-Entry 1-1 +weight: 37 +topics: + - srctest-entries/entry1-1, srctest-related-topic-for-entry1-1 + - srctest-entries/entry1-1-extended +aggregate: topic +--- + +# Title Heading for srctest-Entry 1-1 + + diff --git a/nb-api/src/test/resources/docs-for-testing-only/srctest-topics-b/intesting-topic2-1/intesting-entry-2-1-L.md b/nb-api/src/test/resources/docs-for-testing-only/srctest-topics-b/intesting-topic2-1/intesting-entry-2-1-L.md new file mode 100644 index 000000000..3c35c43ae --- /dev/null +++ b/nb-api/src/test/resources/docs-for-testing-only/srctest-topics-b/intesting-topic2-1/intesting-entry-2-1-L.md @@ -0,0 +1,8 @@ +--- +title: srctest-Entry 2-1-L +weight: 39 +--- + +# Title Heading for srctest-Entry 2-1-L + + diff --git a/nb-api/src/test/resources/docs-for-testing-only/topics-a/entry1-1.md b/nb-api/src/test/resources/docs-for-testing-only/topics-a/entry1-1.md deleted file mode 100644 index 4d5a2c8c9..000000000 --- a/nb-api/src/test/resources/docs-for-testing-only/topics-a/entry1-1.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: Entry 1-1 -weight: 37 -topic: entries/entry1-1 -aggregate: topic ---- - -# Title Heading for Entry 1-1 - -