From 14521bd5b0e5c8b7c7dd01c7d9b038c15fcd94ea Mon Sep 17 00:00:00 2001 From: magnesj Date: Mon, 11 Nov 2013 14:36:59 +0100 Subject: [PATCH] Temporarily fixes for handling communication of more than 2GB data to/from octave --- OctavePlugin/riGetActiveCellProperty.cpp | 4 +- OctavePlugin/riGetGridProperty.cpp | 87 +++++++++++++++++++++--- OctavePlugin/riSetGridProperty.cpp | 13 ++-- 3 files changed, 88 insertions(+), 16 deletions(-) diff --git a/OctavePlugin/riGetActiveCellProperty.cpp b/OctavePlugin/riGetActiveCellProperty.cpp index a936b82d96..81b966f7d0 100644 --- a/OctavePlugin/riGetActiveCellProperty.cpp +++ b/OctavePlugin/riGetActiveCellProperty.cpp @@ -67,7 +67,7 @@ void getActiveCellProperty(Matrix& propertyFrames, const QString &serverName, qu for (size_t tIdx = 0; tIdx < timestepCount; ++tIdx) { - while (socket.bytesAvailable() < (int)byteCount) + while (socket.bytesAvailable() < (qint64)byteCount) { if (!socket.waitForReadyRead(riOctavePlugin::longTimeOutMilliSecs)) { @@ -94,7 +94,7 @@ void getActiveCellProperty(Matrix& propertyFrames, const QString &serverName, qu } #endif - if ((int)byteCount != bytesRead) + if ((qint64)byteCount != bytesRead) { error("Could not read binary double data properly from socket"); octave_stdout << "Active cells: " << activeCellCount << ", Timesteps: " << timestepCount << std::endl; diff --git a/OctavePlugin/riGetGridProperty.cpp b/OctavePlugin/riGetGridProperty.cpp index 3771341644..3b1fbc589f 100644 --- a/OctavePlugin/riGetGridProperty.cpp +++ b/OctavePlugin/riGetGridProperty.cpp @@ -22,7 +22,7 @@ void getGridProperty(NDArray& propertyFrames, const QString &serverName, quint16 QString command; command += "GetGridProperty " + QString::number(caseId) + " " + QString::number(gridIdx) + " " + propertyName + " " + porosityModel; - for (int i = 0; i < requestedTimeSteps.length(); ++i) + for (qint64 i = 0; i < requestedTimeSteps.length(); ++i) { if (i == 0) command += " "; command += QString::number(static_cast(requestedTimeSteps.elem(i)) - 1); // To make the index 0-based @@ -36,7 +36,7 @@ void getGridProperty(NDArray& propertyFrames, const QString &serverName, quint16 // Get response. First wait for the header - while (socket.bytesAvailable() < (int)(4*sizeof(quint64))) + while (socket.bytesAvailable() < (qint64)(4*sizeof(quint64))) { if (!socket.waitForReadyRead(riOctavePlugin::longTimeOutMilliSecs)) { @@ -48,6 +48,7 @@ void getGridProperty(NDArray& propertyFrames, const QString &serverName, quint16 // Read sizes quint64 totalByteCount; + quint64 cellCountI; quint64 cellCountJ; quint64 cellCountK; @@ -59,6 +60,9 @@ void getGridProperty(NDArray& propertyFrames, const QString &serverName, quint16 socketStream >> timestepCount; totalByteCount = cellCountI*cellCountJ*cellCountK*timestepCount*sizeof(double); + + qint64 timestepByteCount = cellCountI*cellCountJ*cellCountK*sizeof(double); + if (!(totalByteCount)) { error ("Could not find the requested data in ResInsight"); @@ -75,29 +79,92 @@ void getGridProperty(NDArray& propertyFrames, const QString &serverName, quint16 propertyFrames.resize(dv); - // Wait for available data - - while (socket.bytesAvailable() < (int)totalByteCount) +#if 1 + // Wait for available data for each timestep, then read data for each timestep + + qint64 totalBytesRead = 0; + + for (size_t tIdx = 0; tIdx < timestepCount; ++tIdx) { - if (!socket.waitForReadyRead(riOctavePlugin::longTimeOutMilliSecs)) + qint64 bytesAvailable = socket.bytesAvailable() ; + + while ( bytesAvailable < (qint64)timestepByteCount) { - error(("Waiting for data : " + socket.errorString()).toLatin1().data()); - return ; + if (!socket.waitForReadyRead(riOctavePlugin::longTimeOutMilliSecs)) + { + error((("Waiting for timestep data number: ") + QString::number(tIdx)+ ": " + socket.errorString()).toLatin1().data()); + octave_stdout << "Cellcount: " << cellCountI*cellCountJ*cellCountK << ", Timesteps: " << timestepCount << std::endl; + return ; + } + + bytesAvailable = socket.bytesAvailable(); + + OCTAVE_QUIT; } + + qint64 bytesRead = 0; + double * internalMatrixData = propertyFrames.fortran_vec(); + + // Raw data transfer. Faster. Not possible when dealing with coarsening + bytesRead = socket.read(((char*)(internalMatrixData)) + tIdx * timestepByteCount, timestepByteCount); + + if ((qint64)timestepByteCount != bytesRead) + { + error("Could not read binary double data properly from socket"); + octave_stdout << "Cellcount: " << cellCountI*cellCountJ*cellCountK << ", Timesteps count: " << timestepCount << std::endl; + octave_stdout << "Timestep : " << tIdx << std::endl; + } + + totalBytesRead += bytesRead; + + OCTAVE_QUIT; + } + + if ((qint64)totalByteCount != totalBytesRead) + { + error("Could not read binary double data properly from socket"); + } + + #else + + // Wait for available data + qint64 bytesAvailable = socket.bytesAvailable() ; + + while (bytesAvailable < (qint64)totalByteCount) + { + octave_stdout << "Waiting for data. Has : " << bytesAvailable << " Needs :" << totalByteCount << std::endl; + if (!socket.waitForReadyRead(riOctavePlugin::shortTimeOutMilliSecs)) + { + //error(("Waiting for data : " + socket.errorString()).toLatin1().data()); + //return ; + } + + bytesAvailable = socket.bytesAvailable() ; OCTAVE_QUIT; } qint64 bytesRead = 0; double * internalMatrixData = propertyFrames.fortran_vec(); +#if 0 + + char* dataBuffer = new char[totalByteCount]; + bytesRead = socket.read(dataBuffer, totalByteCount); + + +#else + // Raw data transfer. Faster. bytesRead = socket.read((char*)(internalMatrixData ), totalByteCount); +#endif - if ((int)totalByteCount != bytesRead) + + if ((qint64)totalByteCount != bytesRead) { error("Could not read binary double data properly from socket"); } - + +#endif QString tmp = QString("riGetGridProperty : Read %1").arg(propertyName); if (caseId < 0) diff --git a/OctavePlugin/riSetGridProperty.cpp b/OctavePlugin/riSetGridProperty.cpp index 04773e8804..f5afe0a0f7 100644 --- a/OctavePlugin/riSetGridProperty.cpp +++ b/OctavePlugin/riSetGridProperty.cpp @@ -64,8 +64,13 @@ void setEclipseProperty(const NDArray& propertyFrames, const QString &hostName, socketStream << (qint64)singleTimeStepByteCount; const double* internalData = propertyFrames.fortran_vec(); - int dataWritten = socket.write((const char *)internalData, singleTimeStepByteCount*timeStepCount); - + qint64 dataWritten = 0; + + for (size_t tsIdx = 0; tsIdx < timeStepCount; ++tsIdx) + { + dataWritten += socket.write(((const char *)internalData) + tsIdx*singleTimeStepByteCount, singleTimeStepByteCount); + } + if (dataWritten == singleTimeStepByteCount*timeStepCount) { QString tmp = QString("riSetGridProperty : Wrote %1").arg(propertyName); @@ -92,8 +97,8 @@ void setEclipseProperty(const NDArray& propertyFrames, const QString &hostName, while(socket.bytesToWrite() && socket.state() == QAbstractSocket::ConnectedState) { - // octave_stdout << "Bytes to write: " << socket.bytesToWrite() << std::endl; - socket.waitForBytesWritten(riOctavePlugin::longTimeOutMilliSecs); + octave_stdout << "Bytes to write: " << socket.bytesToWrite() << std::endl << std::flush; + socket.waitForBytesWritten(2000); OCTAVE_QUIT; }