OilfieldToolsNet/ResInsight
4
0
Fork 0
mirror of https://github.com/OPM/ResInsight.git synced 2025-02-25 18:55:39 -06:00
Code Issues Packages Projects Releases Wiki Activity

Refactor: Extract arrow tools.

Browse Source
This commit is contained in:
Kristian Bendiksen
2024-06-14 15:33:53 +02:00
parent 7125670d8c
commit 71929a234a
5 changed files with 104 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ApplicationLibCode/FileInterface/CMakeLists_files.cmake
View File

@@ -99,6 +99,7 @@ set(SOURCE_GROUP_HEADER_FILES
${CMAKE_CURRENT_LIST_DIR}/RifOsduWellPathReader.h
${CMAKE_CURRENT_LIST_DIR}/RifAsciiDataParseOptions.h
${CMAKE_CURRENT_LIST_DIR}/RifByteArrayArrowRandomAccessFile.h
${CMAKE_CURRENT_LIST_DIR}/RifArrowTools.h
)
set(SOURCE_GROUP_SOURCE_FILES
@@ -197,6 +198,7 @@ set(SOURCE_GROUP_SOURCE_FILES
${CMAKE_CURRENT_LIST_DIR}/RifOsduWellPathReader.cpp
${CMAKE_CURRENT_LIST_DIR}/RifOsduWellLogReader.cpp
${CMAKE_CURRENT_LIST_DIR}/RifByteArrayArrowRandomAccessFile.cpp
${CMAKE_CURRENT_LIST_DIR}/RifArrowTools.cpp
)
list(APPEND CODE_HEADER_FILES ${SOURCE_GROUP_HEADER_FILES})

63
ApplicationLibCode/FileInterface/RifArrowTools.cpp Normal file
View File

@@ -0,0 +1,63 @@
/////////////////////////////////////////////////////////////////////////////////
//
// Copyright (C) 2024 Equinor ASA
//
// ResInsight is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY
// WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE.
//
// See the GNU General Public License at <http://www.gnu.org/licenses/gpl.html>
// for more details.
//
/////////////////////////////////////////////////////////////////////////////////
#include "RifArrowTools.h"
#include "cafAssert.h"
#include <vector>
// #include <arrow/array/array_primitive.h>
// #include <arrow/csv/api.h>
// #include <arrow/io/api.h>
// #include <arrow/scalar.h>
// #include <parquet/arrow/reader.h>
//--------------------------------------------------------------------------------------------------
///
//--------------------------------------------------------------------------------------------------
std::vector<double> RifArrowTools::convertChunkedArrayToStdVector( const std::shared_ptr<arrow::ChunkedArray>& column )
{
auto convertChunkToVector = []( const std::shared_ptr<arrow::Array>& array ) -> std::vector<double>
{
std::vector<double> result;
auto double_array = std::static_pointer_cast<arrow::DoubleArray>( array );
result.resize( double_array->length() );
for ( int64_t i = 0; i < double_array->length(); ++i )
{
result[i] = double_array->Value( i );
}
return result;
};
CAF_ASSERT( column->type()->id() == arrow::Type::DOUBLE );
std::vector<double> result;
// Iterate over each chunk in the column
for ( int i = 0; i < column->num_chunks(); ++i )
{
std::shared_ptr<arrow::Array> chunk = column->chunk( i );
std::vector<double> chunk_vector = convertChunkToVector( chunk );
result.insert( result.end(), chunk_vector.begin(), chunk_vector.end() );
}
return result;
};

34
ApplicationLibCode/FileInterface/RifArrowTools.h Normal file
View File

@@ -0,0 +1,34 @@
/////////////////////////////////////////////////////////////////////////////////
//
// Copyright (C) 2024- Equinor ASA
//
// ResInsight is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY
// WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE.
//
// See the GNU General Public License at <http://www.gnu.org/licenses/gpl.html>
// for more details.
//
/////////////////////////////////////////////////////////////////////////////////
#pragma once
#include <arrow/array/array_primitive.h>
#include <memory>
#include <vector>
//==================================================================================================
//
//
//==================================================================================================
class RifArrowTools
{
public:
static std::vector<double> convertChunkedArrayToStdVector( const std::shared_ptr<arrow::ChunkedArray>& column );
};

36
ApplicationLibCode/FileInterface/RifOsduWellLogReader.cpp
View File

@@ -25,6 +25,8 @@
#include <iostream>
#include <vector>
#include "RifArrowTools.h"
#include <arrow/array/array_primitive.h>
#include <arrow/csv/api.h>
#include <arrow/io/api.h>
@@ -36,38 +38,6 @@
//--------------------------------------------------------------------------------------------------
std::pair<cvf::ref<RigOsduWellLogData>, QString> RifOsduWellLogReader::readWellLogData( const QByteArray& contents )
{
// Function to convert an entire column to std::vector<double>
auto convertColumnToVector = []( const std::shared_ptr<arrow::ChunkedArray>& column ) -> std::vector<double>
{
auto convertChunkToVector = []( const std::shared_ptr<arrow::Array>& array ) -> std::vector<double>
{
std::vector<double> result;
auto double_array = std::static_pointer_cast<arrow::DoubleArray>( array );
result.resize( double_array->length() );
for ( int64_t i = 0; i < double_array->length(); ++i )
{
result[i] = double_array->Value( i );
}
return result;
};
CAF_ASSERT( column->type()->id() == arrow::Type::DOUBLE );
std::vector<double> result;
// Iterate over each chunk in the column
for ( int i = 0; i < column->num_chunks(); ++i )
{
std::shared_ptr<arrow::Array> chunk = column->chunk( i );
std::vector<double> chunk_vector = convertChunkToVector( chunk );
result.insert( result.end(), chunk_vector.begin(), chunk_vector.end() );
}
return result;
};
arrow::MemoryPool* pool = arrow::default_memory_pool();
std::shared_ptr<arrow::io::RandomAccessFile> input = std::make_shared<RifByteArrayArrowRandomAccessFile>( contents );
@@ -93,7 +63,7 @@ std::pair<cvf::ref<RigOsduWellLogData>, QString> RifOsduWellLogReader::readWellL
if ( column->type()->id() == arrow::Type::DOUBLE )
{
std::vector<double> columnVector = convertColumnToVector( column );
std::vector<double> columnVector = RifArrowTools::convertChunkedArrayToStdVector( column );
logData->setValues( QString::fromStdString( columnName ), columnVector );
}
}

35
ApplicationLibCode/FileInterface/RifOsduWellPathReader.cpp
View File

@@ -28,6 +28,7 @@
#include "RiaLogging.h"
#include "RiaTextStringTools.h"
#include "RifArrowTools.h"
#include "RifAsciiDataParseOptions.h"
#include "RifByteArrayArrowRandomAccessFile.h"
#include "RifCsvUserDataParser.h"
@@ -107,38 +108,6 @@ std::pair<cvf::ref<RigWellPath>, QString> RifOsduWellPathReader::parseCsv( const
//--------------------------------------------------------------------------------------------------
std::pair<cvf::ref<RigWellPath>, QString> RifOsduWellPathReader::readWellPathData( const QByteArray& content )
{
// Function to convert an entire column to std::vector<double>
auto convertColumnToVector = []( const std::shared_ptr<arrow::ChunkedArray>& column ) -> std::vector<double>
{
auto convertChunkToVector = []( const std::shared_ptr<arrow::Array>& array ) -> std::vector<double>
{
std::vector<double> result;
auto double_array = std::static_pointer_cast<arrow::DoubleArray>( array );
result.resize( double_array->length() );
for ( int64_t i = 0; i < double_array->length(); ++i )
{
result[i] = double_array->Value( i );
}
return result;
};
CAF_ASSERT( column->type()->id() == arrow::Type::DOUBLE );
std::vector<double> result;
// Iterate over each chunk in the column
for ( int i = 0; i < column->num_chunks(); ++i )
{
std::shared_ptr<arrow::Array> chunk = column->chunk( i );
std::vector<double> chunk_vector = convertChunkToVector( chunk );
result.insert( result.end(), chunk_vector.begin(), chunk_vector.end() );
}
return result;
};
arrow::MemoryPool* pool = arrow::default_memory_pool();
std::shared_ptr<arrow::io::RandomAccessFile> input = std::make_shared<RifByteArrayArrowRandomAccessFile>( content );
@@ -172,7 +141,7 @@ std::pair<cvf::ref<RigWellPath>, QString> RifOsduWellPathReader::readWellPathDat
if ( column->type()->id() == arrow::Type::DOUBLE )
{
std::vector<double> columnVector = convertColumnToVector( column );
std::vector<double> columnVector = RifArrowTools::convertChunkedArrayToStdVector( column );
RiaLogging::debug( QString( "Column name: %1. Size: %2" ).arg( QString::fromStdString( columnName ) ).arg( columnVector.size() ) );
readValues[columnName] = columnVector;
}