From 37a0954fa819ebd93dc183b6b4df2f0681dd574a Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 11 Jan 2022 16:54:33 -0600 Subject: [PATCH] provide complete metadata on bundled markdown files --- .../java/io/nosqlbench/engine/cli/NBCLI.java | 2 +- nb-api/pom.xml | 5 + .../BundledFrontmatterInjector.java | 33 +++++ .../docexporter/BundledMarkdownExporter.java | 35 ++++++ .../docexporter/BundledMarkdownProcessor.java | 10 ++ .../BundledMarkdownZipExporter.java} | 57 ++++----- .../aggregator/MutableFrontMatter.java | 49 ++++++++ .../markdown/aggregator/MutableMarkdown.java | 117 ++++++++++++++++++ .../aggregator/ParsedFrontMatter.java | 51 +++++--- .../markdown/types/BasicFrontMatterInfo.java | 17 +++ .../api/markdown/types/FrontMatterInfo.java | 4 +- .../nb/api/markdown/types/MarkdownInfo.java | 2 - .../docapi/BundledMarkdownExporterTest.java | 1 + nb-api/src/test/resources/testsite1/basic1.md | 6 + 14 files changed, 335 insertions(+), 54 deletions(-) create mode 100644 nb-api/src/main/java/io/nosqlbench/docexporter/BundledFrontmatterInjector.java create mode 100644 nb-api/src/main/java/io/nosqlbench/docexporter/BundledMarkdownExporter.java create mode 100644 nb-api/src/main/java/io/nosqlbench/docexporter/BundledMarkdownProcessor.java rename nb-api/src/main/java/io/nosqlbench/{docapi/BundledMarkdownExporter.java => docexporter/BundledMarkdownZipExporter.java} (60%) create mode 100644 nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableFrontMatter.java create mode 100644 nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableMarkdown.java create mode 100644 nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/BasicFrontMatterInfo.java diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java index bfe2c0c9e..6f087f308 100644 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java @@ -1,6 +1,6 @@ package io.nosqlbench.engine.cli; -import io.nosqlbench.docapi.BundledMarkdownExporter; +import io.nosqlbench.docexporter.BundledMarkdownExporter; import io.nosqlbench.docsys.core.NBWebServerApp; import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogDumperUtility; import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogImporterUtility; diff --git a/nb-api/pom.xml b/nb-api/pom.xml index e16b006ec..2ef6eb2c0 100644 --- a/nb-api/pom.xml +++ b/nb-api/pom.xml @@ -44,6 +44,11 @@ 0.62.2 + + org.yaml + snakeyaml + + net.sf.jopt-simple diff --git a/nb-api/src/main/java/io/nosqlbench/docexporter/BundledFrontmatterInjector.java b/nb-api/src/main/java/io/nosqlbench/docexporter/BundledFrontmatterInjector.java new file mode 100644 index 000000000..6c472daa4 --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/docexporter/BundledFrontmatterInjector.java @@ -0,0 +1,33 @@ +package io.nosqlbench.docexporter; + +import io.nosqlbench.nb.api.markdown.aggregator.MutableMarkdown; + +import java.util.Locale; + +public class BundledFrontmatterInjector implements BundledMarkdownProcessor { + + @Override + public MutableMarkdown apply(MutableMarkdown parsedMarkdown) { + if (parsedMarkdown.getFrontmatter().getWeight()<0) { + String title = parsedMarkdown.getFrontmatter().getTitle(); + parsedMarkdown.getFrontmatter().setWeight(alphaWeightOf(title)); + } + return parsedMarkdown; + } + + private int alphaWeightOf(String name) { + name=name.toLowerCase(Locale.ROOT); + int sum=0; + int pow=26; + for (int i = 0; i < 6; i++) { + if (name.length()>i) { + int ord = name.charAt(i) - 'a'; + double addend = Math.pow(pow, i) * ord; + sum += addend; + } else { + break; + } + } + return sum; + } +} diff --git a/nb-api/src/main/java/io/nosqlbench/docexporter/BundledMarkdownExporter.java b/nb-api/src/main/java/io/nosqlbench/docexporter/BundledMarkdownExporter.java new file mode 100644 index 000000000..155b69848 --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/docexporter/BundledMarkdownExporter.java @@ -0,0 +1,35 @@ +package io.nosqlbench.docexporter; + +import joptsimple.OptionParser; +import joptsimple.OptionSet; +import joptsimple.OptionSpec; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; + +public class BundledMarkdownExporter { + + public static void main(String[] args) { + + final OptionParser parser = new OptionParser(); + + OptionSpec zipfileSpec = parser.accepts("zipfile", "zip file to write to") + .withOptionalArg().ofType(String.class).defaultsTo("exported_docs.zip"); + + OptionSpec helpSpec = parser.acceptsAll(List.of("help", "h", "?"), "Display help").forHelp(); + OptionSet options = parser.parse(args); + if (options.has(helpSpec)) { + try { + parser.printHelpOn(System.out); + } catch (IOException e) { + throw new RuntimeException("Unable to show help:" + e); + } + } + + String zipfile = options.valueOf(zipfileSpec); + + new BundledMarkdownZipExporter(new BundledFrontmatterInjector()).exportDocs(Path.of(zipfile)); + } + +} diff --git a/nb-api/src/main/java/io/nosqlbench/docexporter/BundledMarkdownProcessor.java b/nb-api/src/main/java/io/nosqlbench/docexporter/BundledMarkdownProcessor.java new file mode 100644 index 000000000..25ddab3fa --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/docexporter/BundledMarkdownProcessor.java @@ -0,0 +1,10 @@ +package io.nosqlbench.docexporter; + +import io.nosqlbench.nb.api.markdown.aggregator.MutableMarkdown; + +import java.util.function.Function; + +public interface BundledMarkdownProcessor extends Function { + @Override + MutableMarkdown apply(MutableMarkdown parsedMarkdown); +} diff --git a/nb-api/src/main/java/io/nosqlbench/docapi/BundledMarkdownExporter.java b/nb-api/src/main/java/io/nosqlbench/docexporter/BundledMarkdownZipExporter.java similarity index 60% rename from nb-api/src/main/java/io/nosqlbench/docapi/BundledMarkdownExporter.java rename to nb-api/src/main/java/io/nosqlbench/docexporter/BundledMarkdownZipExporter.java index 13f26a723..671b462fc 100644 --- a/nb-api/src/main/java/io/nosqlbench/docapi/BundledMarkdownExporter.java +++ b/nb-api/src/main/java/io/nosqlbench/docexporter/BundledMarkdownZipExporter.java @@ -1,44 +1,33 @@ -package io.nosqlbench.docapi; +package io.nosqlbench.docexporter; -import joptsimple.OptionParser; -import joptsimple.OptionSet; -import joptsimple.OptionSpec; +import io.nosqlbench.docapi.BundledMarkdownLoader; +import io.nosqlbench.docapi.DocsBinder; +import io.nosqlbench.docapi.DocsNameSpace; +import io.nosqlbench.nb.api.markdown.aggregator.MutableMarkdown; import java.io.File; import java.io.IOException; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.util.List; +import java.util.Locale; +import java.util.function.Function; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; -public class BundledMarkdownExporter { - public static void main(String[] args) { +public class BundledMarkdownZipExporter { - final OptionParser parser = new OptionParser(); + private final BundledMarkdownProcessor[] filters; + private final Function parser = MutableMarkdown::new; - OptionSpec zipfileSpec = parser.accepts("zipfile", "zip file to write to") - .withOptionalArg().ofType(String.class).defaultsTo("exported_docs.zip"); - - OptionSpec helpSpec = parser.acceptsAll(List.of("help", "h", "?"), "Display help").forHelp(); - OptionSet options = parser.parse(args); - if (options.has(helpSpec)) { - try { - parser.printHelpOn(System.out); - } catch (IOException e) { - throw new RuntimeException("Unable to show help:" + e); - } - } - - String zipfile = options.valueOf(zipfileSpec); - - new BundledMarkdownExporter().exportDocs(Path.of(zipfile)); + public BundledMarkdownZipExporter(BundledMarkdownProcessor... filters) { + this.filters = filters; } - private void exportDocs(Path out) { + public void exportDocs(Path out) { ZipOutputStream zipstream; try { OutputStream stream = Files.newOutputStream(out, StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); @@ -66,21 +55,29 @@ public class BundledMarkdownExporter { ZipEntry entry = new ZipEntry(name); - if (Files.isDirectory(p)) { zos.putNextEntry(entry); DirectoryStream stream = Files.newDirectoryStream(p); for (Path path : stream) { addEntry(path,r,zos); } - zos.closeEntry(); } else { entry.setTime(Files.getLastModifiedTime(p).toMillis()); zos.putNextEntry(entry); - byte[] bytes = Files.readAllBytes(p); - zos.write(bytes); - zos.closeEntry(); + + if (p.toString().toLowerCase(Locale.ROOT).endsWith(".md")) { + MutableMarkdown parsed = parser.apply(p); + for (BundledMarkdownProcessor filter : this.filters) { + parsed = filter.apply(parsed); + } + zos.write(parsed.getComposedMarkdown().getBytes(StandardCharsets.UTF_8)); + } else { + byte[] bytes = Files.readAllBytes(p); + zos.write(bytes); + } } + zos.closeEntry(); } + } diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableFrontMatter.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableFrontMatter.java new file mode 100644 index 000000000..0a4fecb1a --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableFrontMatter.java @@ -0,0 +1,49 @@ +package io.nosqlbench.nb.api.markdown.aggregator; + +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.Yaml; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +public class MutableFrontMatter extends LinkedHashMap> { + String WEIGHT = "weight"; + String TITLE = "title"; + + MutableFrontMatter(Map> data) { + this.putAll(data); + } + + public String getTitle() { + assertMaxSingleValued(TITLE); + return Optional.ofNullable(get(TITLE)).map(l -> l.get(0)).orElse(null); + } + + public int getWeight() { + assertMaxSingleValued(WEIGHT); + return Optional.ofNullable(get(WEIGHT)).map(l -> l.get(0)).map(Integer::parseInt).orElse(0); + } + + public void setTitle(String title) { + put(TITLE,List.of(title)); + } + + public void setWeight(int weight) { + put(WEIGHT,List.of(String.valueOf(weight))); + } + + private void assertMaxSingleValued(String fieldname) { + if (containsKey(fieldname) && get(fieldname).size()>1) { + throw new RuntimeException("Field '" + fieldname + "' can only have zero or one value. It is single-valued."); + } + } + + public String asYaml() { + DumperOptions options = new DumperOptions(); + options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + Yaml yaml = new Yaml(options); + return yaml.dump(Map.of(TITLE,getTitle(),WEIGHT,getWeight())); + } +} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableMarkdown.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableMarkdown.java new file mode 100644 index 000000000..9a52456f7 --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableMarkdown.java @@ -0,0 +1,117 @@ +package io.nosqlbench.nb.api.markdown.aggregator; + +import com.vladsch.flexmark.ast.Heading; +import com.vladsch.flexmark.ast.WhiteSpace; +import com.vladsch.flexmark.ext.yaml.front.matter.AbstractYamlFrontMatterVisitor; +import com.vladsch.flexmark.ext.yaml.front.matter.YamlFrontMatterBlock; +import com.vladsch.flexmark.util.ast.BlankLine; +import com.vladsch.flexmark.util.ast.Document; +import com.vladsch.flexmark.util.ast.Node; +import io.nosqlbench.nb.api.markdown.FlexParser; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class MutableMarkdown { + private final static Logger logger = LogManager.getLogger(MarkdownDocs.class); + + private MutableFrontMatter frontMatter; + private final String rawMarkdown; + private final Path path; + private Heading firstHeading; + + public MutableMarkdown(Path path) { + try { + this.path = path; + this.rawMarkdown = Files.readString(path); + parseStructure(rawMarkdown); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void parseStructure(String rawMarkdown) { + AbstractYamlFrontMatterVisitor v = new AbstractYamlFrontMatterVisitor(); + Document parsed = FlexParser.parser.parse(rawMarkdown); + v.visit(parsed); + Map> data = v.getData(); + this.frontMatter = new MutableFrontMatter(data); + + if (frontMatter.getTitle()==null || frontMatter.getTitle().isEmpty()) { + Node node = parsed.getFirstChild(); + while (node!=null) { + if (node instanceof Heading) { + this.frontMatter.setTitle(((Heading) node).getText().toString()); + break; + } else if (node instanceof BlankLine) { + } else if (node instanceof WhiteSpace) { + } else if (node instanceof YamlFrontMatterBlock) { + } else { + throw new RuntimeException("The markdown file at '" + this.path.toString() + "' must have an initial heading as a title, before any other element, but found:" + node.getClass().getSimpleName()); + } + node=node.getNext(); + } + } + if (frontMatter.getTitle()==null || frontMatter.getTitle().isEmpty()) { + throw new RuntimeException("The markdown file at '" + this.path.toString() + "' has no heading to use as a title."); + } + } + + public Path getPath() { + return path; + } + + public String getBody() { + for (String boundary : List.of("---\n", "+++\n")) { + if (rawMarkdown.startsWith(boundary)) { + int end = rawMarkdown.indexOf(boundary, 3); + if (end>=0) { + return rawMarkdown.substring(end+4); + } else { + throw new RuntimeException("Unable to find matching boundaries in " + path.toString() + ": " + boundary); + } + } + } + return rawMarkdown; + } + + public MutableFrontMatter getFrontmatter() { + return frontMatter; + } + + @Override + public String toString() { + return getClass().getSimpleName() + "/" + + frontMatter.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + MutableMarkdown that = (MutableMarkdown) o; + return Objects.equals(frontMatter, that.frontMatter) && + Objects.equals(rawMarkdown, that.rawMarkdown); + } + + @Override + public int hashCode() { + return Objects.hash(frontMatter, rawMarkdown); + } + + public String getComposedMarkdown() { + StringBuilder sb = new StringBuilder(); + sb.append("---\n"); + sb.append(frontMatter.asYaml()); + sb.append("---\n"); + + sb.append(getBody()); + return sb.toString(); + } +} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ParsedFrontMatter.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ParsedFrontMatter.java index e87173523..29aef5854 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ParsedFrontMatter.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ParsedFrontMatter.java @@ -1,5 +1,6 @@ package io.nosqlbench.nb.api.markdown.aggregator; +import io.nosqlbench.nb.api.markdown.types.BasicFrontMatterInfo; import io.nosqlbench.nb.api.markdown.types.DocScope; import io.nosqlbench.nb.api.markdown.types.FrontMatterInfo; import org.apache.logging.log4j.LogManager; @@ -14,34 +15,40 @@ public class ParsedFrontMatter implements FrontMatterInfo { private final static Logger logger = LogManager.getLogger(ParsedFrontMatter.class); - private final Map> data; + private final Map> data = new LinkedHashMap<>(); public ParsedFrontMatter(Map> data) { - this.data = data; + this.data.putAll(data); } @Override public String getTitle() { - List titles = data.get(FrontMatterInfo.TITLE); + List titles = data.get(BasicFrontMatterInfo.TITLE); if (titles==null) { return ""; } - if (titles.size()!=1) { - throw new InvalidParameterException(FrontMatterInfo.TITLE + " can only contain a single value."); + if (titles.size()>1) { + throw new InvalidParameterException(BasicFrontMatterInfo.TITLE + " can only contain a single value."); } - return titles.get(0); + if (titles.size()==1) { + return titles.get(0); + } + return ""; } @Override public int getWeight() { - List weights = data.get(FrontMatterInfo.WEIGHT); + List weights = data.get(BasicFrontMatterInfo.WEIGHT); if (weights==null) { return 0; } - if (weights.size()!=1) { - throw new InvalidParameterException(FrontMatterInfo.WEIGHT + " can only contain a single value."); + if (weights.size()>1) { + throw new InvalidParameterException(BasicFrontMatterInfo.WEIGHT + " can only contain a single value."); } - return Integer.parseInt(weights.get(0)); + if (weights.size()==1) { + return Integer.parseInt(weights.get(0)); + } + return 0; } @Override @@ -93,22 +100,21 @@ public class ParsedFrontMatter implements FrontMatterInfo { return scopeNames.stream().map(DocScope::valueOf).collect(Collectors.toSet()); } - public List getDiagnostics() { - List warnings = new ArrayList<>(); + @Override + public List getDiagnostics(List buffer) { for (String propname : data.keySet()) { if (!FrontMatterInfo.FrontMatterKeyWords.contains(propname)) { - warnings.add("unrecognized frontm atter property " + propname); + buffer.add("unrecognized frontmatter property " + propname); } } - return warnings; + return buffer; } - - public void setTopics(Set newTopics) { - // TODO: allow functional version of this -// this.data.put(FrontMatterInfo.TOPICS,newTopics); + public List getDiagnostics() { + return getDiagnostics(new ArrayList<>()); } + public ParsedFrontMatter withTopics(List assigning) { HashMap> newmap = new HashMap<>(); newmap.putAll(this.data); @@ -142,4 +148,13 @@ public class ParsedFrontMatter implements FrontMatterInfo { public int hashCode() { return Objects.hash(data); } + + public void setTitle(String title) { + this.data.put(FrontMatterInfo.TITLE,List.of(title)); + } + + public void setWeight(int weight) { + data.put(FrontMatterInfo.WEIGHT,List.of(String.valueOf(weight))); + } + } diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/BasicFrontMatterInfo.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/BasicFrontMatterInfo.java new file mode 100644 index 000000000..4f952dd79 --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/BasicFrontMatterInfo.java @@ -0,0 +1,17 @@ +package io.nosqlbench.nb.api.markdown.types; + +public interface BasicFrontMatterInfo { + + String WEIGHT = "weight"; + String TITLE = "title"; + + /** + * @return A title for the given markdown source file. + */ + String getTitle(); + + /** + * @return A weight for the given markdown source file. + */ + int getWeight(); +} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/FrontMatterInfo.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/FrontMatterInfo.java index a3e5c0d6c..6daaab3f9 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/FrontMatterInfo.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/FrontMatterInfo.java @@ -9,13 +9,11 @@ import java.util.regex.Pattern; * If the markdown source file does not contain the metadata requested, then reasonable non-null * defaults must be provided. */ -public interface FrontMatterInfo { +public interface FrontMatterInfo extends BasicFrontMatterInfo, HasDiagnostics { String SCOPES = "scopes"; String AGGREGATE = "aggregate"; String TOPICS = "topics"; - String WEIGHT = "weight"; - String TITLE = "title"; String INCLUDED = "included"; Set FrontMatterKeyWords = diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/MarkdownInfo.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/MarkdownInfo.java index 0d741198e..2cbb564fa 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/MarkdownInfo.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/MarkdownInfo.java @@ -1,7 +1,5 @@ package io.nosqlbench.nb.api.markdown.types; -import io.nosqlbench.nb.api.markdown.aggregator.CompositeMarkdownInfo; -import io.nosqlbench.nb.api.markdown.types.FrontMatterInfo; import org.jetbrains.annotations.NotNull; import java.nio.file.Path; diff --git a/nb-api/src/test/java/io/nosqlbench/docapi/BundledMarkdownExporterTest.java b/nb-api/src/test/java/io/nosqlbench/docapi/BundledMarkdownExporterTest.java index edf6f79f7..c31ba29e2 100644 --- a/nb-api/src/test/java/io/nosqlbench/docapi/BundledMarkdownExporterTest.java +++ b/nb-api/src/test/java/io/nosqlbench/docapi/BundledMarkdownExporterTest.java @@ -1,5 +1,6 @@ package io.nosqlbench.docapi; +import io.nosqlbench.docexporter.BundledMarkdownExporter; import org.junit.jupiter.api.Test; public class BundledMarkdownExporterTest { diff --git a/nb-api/src/test/resources/testsite1/basic1.md b/nb-api/src/test/resources/testsite1/basic1.md index 2d147d55b..9c17b0c32 100644 --- a/nb-api/src/test/resources/testsite1/basic1.md +++ b/nb-api/src/test/resources/testsite1/basic1.md @@ -1,3 +1,9 @@ +--- +RandomFrontMatter1: value +--- + +# Heading + ## Basic Markdown File - item 1.