Merge pull request #706 from derrickCos/compaction-test

Added op statements to generate sstable and rainbowtable

adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4RainbowTableDispenser.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.adapter.cqld4.opdispensers;
+
+import com.datastax.oss.driver.api.core.CqlSession;
+import com.datastax.oss.driver.api.core.cql.Statement;
+import io.nosqlbench.adapter.cqld4.optypes.Cqld4RainbowTableOp;
+import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
+import io.nosqlbench.engine.api.templating.ParsedOp;
+
+import java.util.function.LongFunction;
+
+public class CqlD4RainbowTableDispenser extends BaseOpDispenser<Cqld4RainbowTableOp> {
+   
+    private final LongFunction<Statement> stmtFunc;
+    private final LongFunction<String> targetFunction;
+
+    public CqlD4RainbowTableDispenser(DriverAdapter adapter, LongFunction<CqlSession> sessionFunc, LongFunction<String> targetFunction, ParsedOp cmd) {
+        super(adapter, sessionFunc,cmd);
+        this.targetFunction=targetFunction;
+        this.tableFunc =createTableFunc(cmd);
+    }
+
+    protected LongFunction<Statement> createTableFunc(ParsedOp op) {
+        
+    }
+
+    @Override
+    public Cqld4RainbowTableOp apply(long value) {
+        return new Cqld4RainbowTableOp(
+            getSessionFunc().apply(value),
+            (RainbowTable) stmtFunc.apply(value),
+            getMaxPages(),
+            isRetryReplace()
+        );
+    }
+}

adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4SsTableDispenser.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.adapter.cqld4.opdispensers;
+
+import com.datastax.oss.driver.api.core.CqlSession;
+import com.datastax.oss.driver.api.core.cql.Statement;
+import io.nosqlbench.adapter.cqld4.optypes.Cqld4SsTableOp;
+import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
+import io.nosqlbench.engine.api.templating.ParsedOp;
+
+import java.util.function.LongFunction;
+
+public class CqlD4SsTableDispenser extends BaseOpDispenser<Cqld4SsTableOp> {
+    
+    private final LongFunction<Statement> stmtFunc;
+    private final LongFunction<String> targetFunction;
+
+    public CqlD4SsTableDispenser(DriverAdapter adapter, LongFunction<CqlSession> sessionFunc, LongFunction<String> targetFunction, ParsedOp cmd) {
+        super(adapter, sessionFunc,cmd);
+        this.targetFunction=targetFunction;
+        this.tableFunc =createTableFunc(cmd);
+    }
+
+    protected LongFunction<Statement> createTableFunc(ParsedOp op) {
+    
+    }
+
+    @Override
+    public Cqld4SsTableOp apply(long value) {
+        return new CqlD4SsTable(
+            getSessionFunc().apply(value),
+            (SsTable) stmtFunc.apply(value),
+            getMaxPages(),
+            isRetryReplace()
+        );
+    }
+}

adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4RainbowTableMapper.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.adapter.cqld4.opmappers;
+
+import com.datastax.oss.driver.api.core.CqlSession;
+import io.nosqlbench.adapter.cqld4.opdispensers.CqlD4RainbowTableDispenser;
+import io.nosqlbench.adapter.cqld4.optypes.Cqld4CqlOp;
+import io.nosqlbench.engine.api.activityimpl.OpDispenser;
+import io.nosqlbench.engine.api.activityimpl.OpMapper;
+import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
+import io.nosqlbench.engine.api.templating.ParsedOp;
+
+import java.util.function.LongFunction;
+
+public class CqlD4RainbowTableMapper implements OpMapper<Cqld4RainbowTableOp> {
+    private final LongFunction<CqlSession> sessionFunc;
+    private final LongFunction<String> targetFunction;
+    private final DriverAdapter adapter;
+
+    public CqlD4RainbowTableMapper(DriverAdapter adapter, LongFunction<CqlSession> sessionFunc, LongFunction<String> targetFunction) {
+        this.sessionFunc = sessionFunc;
+        this.targetFunction = targetFunction;
+        this.adapter = adapter;
+    }
+
+    @Override
+    public OpDispenser<? extends Cqld4CqlOp> apply(ParsedOp op) {
+        return new CqlD4RainbowTableDispenser(adapter, sessionFunc,targetFunction, op);
+    }
+}

adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4SsTableMapper.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.adapter.cqld4.opmappers;
+
+import com.datastax.oss.driver.api.core.CqlSession;
+import io.nosqlbench.adapter.cqld4.opdispensers.Cqld4SsTableDispenser;
+import io.nosqlbench.adapter.cqld4.optypes.Cqld4CqlOp;
+import io.nosqlbench.engine.api.activityimpl.OpDispenser;
+import io.nosqlbench.engine.api.activityimpl.OpMapper;
+import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
+import io.nosqlbench.engine.api.templating.ParsedOp;
+
+import java.util.function.LongFunction;
+
+public class Cqld4SsTableMapper implements OpMapper<Cqld4SsTableOp> {
+    private final LongFunction<CqlSession> sessionFunc;
+    private final LongFunction<String> targetFunction;
+    private final DriverAdapter adapter;
+
+    public CqlD4SsTableMapper(DriverAdapter adapter, LongFunction<CqlSession> sessionFunc, LongFunction<String> targetFunction) {
+        this.sessionFunc = sessionFunc;
+        this.targetFunction = targetFunction;
+        this.adapter = adapter;
+    }
+
+    @Override
+    public OpDispenser<? extends Cqld4CqlOp> apply(ParsedOp op) {
+        return new CqlD4SsTableDispenser(adapter, sessionFunc,targetFunction, op);
+    }
+}

adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4CoreOpMapper.java

@@ -74,6 +74,8 @@ public class Cqld4CoreOpMapper implements OpMapper<Op> {
             case prepared -> new CqlD4PreparedStmtMapper(adapter, sessionFunc, target).apply(op);
             case gremlin -> new Cqld4GremlinOpMapper(adapter, sessionFunc, target.targetFunction).apply(op);
             case fluent -> new Cqld4FluentGraphOpMapper(adapter, sessionFunc, target).apply(op);
+            case rainbow -> new CqlD4RainbowTableMapper(adapter, sessionFunc, target.targetFunction).apply(op);
+            case sst -> new CqlD4SsTableMapper(adapter, sessionFunc, target.targetFunction).apply(op);
         };
     }
 

adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4RainbowTableOp.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.adapter.cqld4.optypes;
+
+import com.datastax.oss.driver.api.core.CqlSession;
+import io.nosqlbench.adapter.cqld4.RSProcessors;
+
+// Need to create RainbowTableStatement
+public class Cqld4RainbowTableOp implements CycleOp<ResultSet>, VariableCapture, OpGenerator, OpResultSize {
+    private final CqlSession session;
+    private final RainbowTableStatement stmt;
+
+    // Rename ResultSet to something more appropriate
+    public final ResultSet apply(long cycle) {
+        // TODO: actually write to sstables
+        // sstable passed to shared memory object
+    }
+    
+    public Cqld4RainbowTableOp(CqlSession session, RainbowTableStatement stmt, int maxpages, boolean retryreplace) {
+        // 
+    }
+
+}

adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4SsTableOp.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.adapter.cqld4.optypes;
+
+import com.datastax.oss.driver.api.core.CqlSession;
+import io.nosqlbench.adapter.cqld4.RSProcessors;
+
+// Need to create SsTableStatement
+public class Cqld4SsTableOp implements CycleOp<ResultSet>, VariableCapture, OpGenerator, OpResultSize  {
+    private final CqlSession session;
+    private final SsTableStatement stmt;
+
+    // Rename ResultSet to something more appropriate
+    public final ResultSet apply(long cycle) {
+        // TODO: actually write to sstables
+        // sstable passed to shared memory object
+    }
+    
+    public Cqld4SsTableOp(CqlSession session, SsTableStatement stmt, int maxpages, boolean retryreplace) {
+
+    }
+
+}

cqlsstable_example.yaml

@@ -0,0 +1,46 @@
+# nb -v run driver=cql yaml=cql-iot tags=phase:schema host=dsehost
+description: |
+  put workload descript here
+scenarios:
+  default:
+    sample: run foo
+
+params:
+  # think about what params we wanna supply and iterate
+  instrument: TEMPLATE(instrument,false)
+
+bindings:
+  machine_id: Mod(<<sources:10000>>); ToHashedUUID() -> java.util.UUID
+  sensor_name: HashedLineToString('data/variable_words.txt')
+  time: Mul(<<timespeed:100>>L); Div(<<sources:10000>>L); ToDate()
+  cell_timestamp: Mul(<<timespeed:100>>L); Div(<<sources:10000>>L); Mul(1000L)
+  sensor_value: Normal(0.0,5.0); Add(100.0) -> double
+  station_id: Div(<<sources:10000>>);Mod(<<stations:100>>); ToHashedUUID() -> java.util.UUID
+  data: HashedFileExtractToString('data/lorem_ipsum_full.txt',800,1200)
+blocks:
+  schema:
+    params:
+      prepared: false
+    ops:
+      create-table: |
+        create table if not exists <<keyspace:baselines>>.<<table:iot>> (
+        machine_id UUID,     // source machine
+        sensor_name text,    // sensor name
+        time timestamp,      // timestamp of collection
+        sensor_value double, //
+        station_id UUID,     // source location
+        data text,
+        PRIMARY KEY ((machine_id, sensor_name), time)
+        );
+  main-write:
+    params:
+      ratio: <<write_ratio:9>>
+      cl: <<write_cl:LOCAL_QUORUM>>
+      idempotent: true
+      instrument: TEMPLATE(instrument-writes,TEMPLATE(instrument,false))
+    ops:
+      insert-main: |
+        insert into <<keyspace:baselines>>.<<table:iot>>
+        (machine_id, sensor_name, time, sensor_value, station_id, data)
+        values ({machine_id}, {sensor_name}, {time}, {sensor_value}, {station_id}, {data})
+        using timestamp {cell_timestamp}