--- libgig/trunk/src/RIFF.cpp 2019/02/22 12:12:50 3481 +++ libgig/trunk/src/RIFF.cpp 2021/06/12 13:51:10 3919 @@ -2,7 +2,7 @@ * * * libgig - C++ cross-platform Gigasampler format file access library * * * - * Copyright (C) 2003-2019 by Christian Schoenebeck * + * Copyright (C) 2003-2021 by Christian Schoenebeck * * * * * * This library is free software; you can redistribute it and/or modify * @@ -52,6 +52,25 @@ return sPath; } + inline static bool _isValidHandle(File::Handle handle) { + #if defined(WIN32) + return handle != INVALID_HANDLE_VALUE; + #else + return handle; + #endif + } + + inline static void _close(File::Handle handle) { + if (!_isValidHandle(handle)) return; + #if POSIX + close(handle); + #elif defined(WIN32) + CloseHandle(handle); + #else + fclose(handle); + #endif + } + // *************** progress_t *************** @@ -129,7 +148,7 @@ #if DEBUG_RIFF std::cout << "Chunk::Chunk(File* pFile)" << std::endl; #endif // DEBUG_RIFF - ullPos = 0; + chunkPos.ullPos = 0; pParent = NULL; pChunkData = NULL; ullCurrentChunkSize = 0; @@ -146,7 +165,7 @@ this->pFile = pFile; ullStartPos = StartPos + CHUNK_HEADER_SIZE(pFile->FileOffsetSize); pParent = Parent; - ullPos = 0; + chunkPos.ullPos = 0; pChunkData = NULL; ullCurrentChunkSize = 0; ullNewChunkSize = 0; @@ -158,7 +177,7 @@ this->pFile = pFile; ullStartPos = 0; // arbitrary usually, since it will be updated when we write the chunk this->pParent = pParent; - ullPos = 0; + chunkPos.ullPos = 0; pChunkData = NULL; ChunkID = uiChunkID; ullChunkDataSize = 0; @@ -176,21 +195,24 @@ #endif // DEBUG_RIFF ChunkID = 0; ullNewChunkSize = ullCurrentChunkSize = 0; + + const File::Handle hRead = pFile->FileHandle(); + #if POSIX - if (lseek(pFile->hFileRead, filePos, SEEK_SET) != -1) { - read(pFile->hFileRead, &ChunkID, 4); - read(pFile->hFileRead, &ullCurrentChunkSize, pFile->FileOffsetSize); + if (lseek(hRead, filePos, SEEK_SET) != -1) { + read(hRead, &ChunkID, 4); + read(hRead, &ullCurrentChunkSize, pFile->FileOffsetSize); #elif defined(WIN32) LARGE_INTEGER liFilePos; liFilePos.QuadPart = filePos; - if (SetFilePointerEx(pFile->hFileRead, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN)) { + if (SetFilePointerEx(hRead, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN)) { DWORD dwBytesRead; - ReadFile(pFile->hFileRead, &ChunkID, 4, &dwBytesRead, NULL); - ReadFile(pFile->hFileRead, &ullCurrentChunkSize, pFile->FileOffsetSize, &dwBytesRead, NULL); + ReadFile(hRead, &ChunkID, 4, &dwBytesRead, NULL); + ReadFile(hRead, &ullCurrentChunkSize, pFile->FileOffsetSize, &dwBytesRead, NULL); #else - if (!fseeko(pFile->hFileRead, filePos, SEEK_SET)) { - fread(&ChunkID, 4, 1, pFile->hFileRead); - fread(&ullCurrentChunkSize, pFile->FileOffsetSize, 1, pFile->hFileRead); + if (!fseeko(hRead, filePos, SEEK_SET)) { + fread(&ChunkID, 4, 1, hRead); + fread(&ullCurrentChunkSize, pFile->FileOffsetSize, 1, hRead); #endif // POSIX #if WORDS_BIGENDIAN if (ChunkID == CHUNK_ID_RIFF) { @@ -236,23 +258,25 @@ swapBytes_64(&ullNewChunkSize); } + const File::Handle hWrite = pFile->FileWriteHandle(); + #if POSIX - if (lseek(pFile->hFileWrite, filePos, SEEK_SET) != -1) { - write(pFile->hFileWrite, &uiNewChunkID, 4); - write(pFile->hFileWrite, &ullNewChunkSize, pFile->FileOffsetSize); + if (lseek(hWrite, filePos, SEEK_SET) != -1) { + write(hWrite, &uiNewChunkID, 4); + write(hWrite, &ullNewChunkSize, pFile->FileOffsetSize); } #elif defined(WIN32) LARGE_INTEGER liFilePos; liFilePos.QuadPart = filePos; - if (SetFilePointerEx(pFile->hFileWrite, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN)) { + if (SetFilePointerEx(hWrite, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN)) { DWORD dwBytesWritten; - WriteFile(pFile->hFileWrite, &uiNewChunkID, 4, &dwBytesWritten, NULL); - WriteFile(pFile->hFileWrite, &ullNewChunkSize, pFile->FileOffsetSize, &dwBytesWritten, NULL); + WriteFile(hWrite, &uiNewChunkID, 4, &dwBytesWritten, NULL); + WriteFile(hWrite, &ullNewChunkSize, pFile->FileOffsetSize, &dwBytesWritten, NULL); } #else - if (!fseeko(pFile->hFileWrite, filePos, SEEK_SET)) { - fwrite(&uiNewChunkID, 4, 1, pFile->hFileWrite); - fwrite(&ullNewChunkSize, pFile->FileOffsetSize, 1, pFile->hFileWrite); + if (!fseeko(hWrite, filePos, SEEK_SET)) { + fwrite(&uiNewChunkID, 4, 1, hWrite); + fwrite(&ullNewChunkSize, pFile->FileOffsetSize, 1, hWrite); } #endif // POSIX } @@ -266,6 +290,42 @@ } /** + * This is an internal-only method which must not be used by any application + * and might change at any time. + * + * Returns a reference (memory location) of the chunk's current file + * (read/write) position variable which depends on the current value of + * File::IsIOPerThread(). + */ + file_offset_t& Chunk::GetPosUnsafeRef() { + if (!pFile->IsIOPerThread()) return chunkPos.ullPos; + const std::thread::id tid = std::this_thread::get_id(); + return chunkPos.byThread[tid]; + } + + /** + * Current read/write position within the chunk data body (starting with 0). + * + * @see File::IsIOPerThread() for multi-threaded streaming + */ + file_offset_t Chunk::GetPos() const { + if (!pFile->IsIOPerThread()) return chunkPos.ullPos; + const std::thread::id tid = std::this_thread::get_id(); + std::lock_guard lock(chunkPos.mutex); + return chunkPos.byThread[tid]; + } + + /** + * Current, actual offset in file of current chunk data body read/write + * position. + * + * @see File::IsIOPerThread() for multi-threaded streaming + */ + file_offset_t Chunk::GetFilePos() const { + return ullStartPos + GetPos(); + } + + /** * Sets the position within the chunk body, thus within the data portion * of the chunk (in bytes). * @@ -276,27 +336,30 @@ * @param Whence - optional: defines to what \a Where relates to, * if omitted \a Where relates to beginning of the chunk * data + * @see File::IsIOPerThread() for multi-threaded streaming */ file_offset_t Chunk::SetPos(file_offset_t Where, stream_whence_t Whence) { #if DEBUG_RIFF std::cout << "Chunk::SetPos(file_offset_t,stream_whence_t)" << std::endl; #endif // DEBUG_RIFF + std::lock_guard lock(chunkPos.mutex); + file_offset_t& pos = GetPosUnsafeRef(); switch (Whence) { case stream_curpos: - ullPos += Where; + pos += Where; break; case stream_end: - ullPos = ullCurrentChunkSize - 1 - Where; + pos = ullCurrentChunkSize - 1 - Where; break; case stream_backward: - ullPos -= Where; + pos -= Where; break; case stream_start: default: - ullPos = Where; + pos = Where; break; } - if (ullPos > ullCurrentChunkSize) ullPos = ullCurrentChunkSize; - return ullPos; + if (pos > ullCurrentChunkSize) pos = ullCurrentChunkSize; + return pos; } /** @@ -308,12 +371,14 @@ * of the chunk data. * * @returns number of bytes left to read + * @see File::IsIOPerThread() for multi-threaded streaming */ file_offset_t Chunk::RemainingBytes() const { #if DEBUG_RIFF std::cout << "Chunk::Remainingbytes()=" << ullCurrentChunkSize - ullPos << std::endl; #endif // DEBUG_RIFF - return (ullCurrentChunkSize > ullPos) ? ullCurrentChunkSize - ullPos : 0; + const file_offset_t pos = GetPos(); + return (ullCurrentChunkSize > pos) ? ullCurrentChunkSize - pos : 0; } /** @@ -339,20 +404,21 @@ * - RIFF::stream_end_reached : * already reached the end of the chunk data, no more reading * possible without SetPos() + * + * @see File::IsIOPerThread() for multi-threaded streaming */ stream_state_t Chunk::GetState() const { #if DEBUG_RIFF std::cout << "Chunk::GetState()" << std::endl; #endif // DEBUG_RIFF - #if POSIX - if (pFile->hFileRead == 0) return stream_closed; - #elif defined (WIN32) - if (pFile->hFileRead == INVALID_HANDLE_VALUE) + + const File::Handle hRead = pFile->FileHandle(); + + if (!_isValidHandle(hRead)) return stream_closed; - #else - if (pFile->hFileRead == NULL) return stream_closed; - #endif // POSIX - if (ullPos < ullCurrentChunkSize) return stream_ready; + + const file_offset_t pos = GetPos(); + if (pos < ullCurrentChunkSize) return stream_ready; else return stream_end_reached; } @@ -370,17 +436,23 @@ * @param WordSize size of each data word to read * @returns number of successfully read data words or 0 if end * of file reached or error occurred + * @see File::IsIOPerThread() for multi-threaded streaming */ file_offset_t Chunk::Read(void* pData, file_offset_t WordCount, file_offset_t WordSize) { #if DEBUG_RIFF std::cout << "Chunk::Read(void*,file_offset_t,file_offset_t)" << std::endl; #endif // DEBUG_RIFF //if (ulStartPos == 0) return 0; // is only 0 if this is a new chunk, so nothing to read (yet) - if (ullPos >= ullCurrentChunkSize) return 0; - if (ullPos + WordCount * WordSize >= ullCurrentChunkSize) WordCount = (ullCurrentChunkSize - ullPos) / WordSize; + const file_offset_t pos = GetPos(); + if (pos >= ullCurrentChunkSize) return 0; + if (pos + WordCount * WordSize >= ullCurrentChunkSize) + WordCount = (ullCurrentChunkSize - pos) / WordSize; + + const File::Handle hRead = pFile->FileHandle(); + #if POSIX - if (lseek(pFile->hFileRead, ullStartPos + ullPos, SEEK_SET) < 0) return 0; - ssize_t readWords = read(pFile->hFileRead, pData, WordCount * WordSize); + if (lseek(hRead, ullStartPos + pos, SEEK_SET) < 0) return 0; + ssize_t readWords = read(hRead, pData, WordCount * WordSize); if (readWords < 1) { #if DEBUG_RIFF std::cerr << "POSIX read() failed: " << strerror(errno) << std::endl << std::flush; @@ -390,16 +462,16 @@ readWords /= WordSize; #elif defined(WIN32) LARGE_INTEGER liFilePos; - liFilePos.QuadPart = ullStartPos + ullPos; - if (!SetFilePointerEx(pFile->hFileRead, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN)) + liFilePos.QuadPart = ullStartPos + pos; + if (!SetFilePointerEx(hRead, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN)) return 0; DWORD readWords; - ReadFile(pFile->hFileRead, pData, WordCount * WordSize, &readWords, NULL); //FIXME: does not work for reading buffers larger than 2GB (even though this should rarely be the case in practice) + ReadFile(hRead, pData, WordCount * WordSize, &readWords, NULL); //FIXME: does not work for reading buffers larger than 2GB (even though this should rarely be the case in practice) if (readWords < 1) return 0; readWords /= WordSize; #else // standard C functions - if (fseeko(pFile->hFileRead, ullStartPos + ullPos, SEEK_SET)) return 0; - file_offset_t readWords = fread(pData, WordSize, WordCount, pFile->hFileRead); + if (fseeko(hRead, ullStartPos + pos, SEEK_SET)) return 0; + file_offset_t readWords = fread(pData, WordSize, WordCount, hRead); #endif // POSIX if (!pFile->bEndianNative && WordSize != 1) { switch (WordSize) { @@ -440,11 +512,14 @@ * @throws RIFF::Exception if write operation would exceed current * chunk size or any IO error occurred * @see Resize() + * @see File::IsIOPerThread() for multi-threaded streaming */ file_offset_t Chunk::Write(void* pData, file_offset_t WordCount, file_offset_t WordSize) { - if (pFile->Mode != stream_mode_read_write) + const File::HandlePair io = pFile->FileHandlePair(); + if (io.Mode != stream_mode_read_write) throw Exception("Cannot write data to chunk, file has to be opened in read+write mode first"); - if (ullPos >= ullCurrentChunkSize || ullPos + WordCount * WordSize > ullCurrentChunkSize) + const file_offset_t pos = GetPos(); + if (pos >= ullCurrentChunkSize || pos + WordCount * WordSize > ullCurrentChunkSize) throw Exception("End of chunk reached while trying to write data"); if (!pFile->bEndianNative && WordSize != 1) { switch (WordSize) { @@ -467,30 +542,30 @@ } } #if POSIX - if (lseek(pFile->hFileWrite, ullStartPos + ullPos, SEEK_SET) < 0) { - throw Exception("Could not seek to position " + ToString(ullPos) + - " in chunk (" + ToString(ullStartPos + ullPos) + " in file)"); + if (lseek(io.hWrite, ullStartPos + pos, SEEK_SET) < 0) { + throw Exception("Could not seek to position " + ToString(pos) + + " in chunk (" + ToString(ullStartPos + pos) + " in file)"); } - ssize_t writtenWords = write(pFile->hFileWrite, pData, WordCount * WordSize); + ssize_t writtenWords = write(io.hWrite, pData, WordCount * WordSize); if (writtenWords < 1) throw Exception("POSIX IO Error while trying to write chunk data"); writtenWords /= WordSize; #elif defined(WIN32) LARGE_INTEGER liFilePos; - liFilePos.QuadPart = ullStartPos + ullPos; - if (!SetFilePointerEx(pFile->hFileWrite, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN)) { - throw Exception("Could not seek to position " + ToString(ullPos) + - " in chunk (" + ToString(ullStartPos + ullPos) + " in file)"); + liFilePos.QuadPart = ullStartPos + pos; + if (!SetFilePointerEx(io.hWrite, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN)) { + throw Exception("Could not seek to position " + ToString(pos) + + " in chunk (" + ToString(ullStartPos + pos) + " in file)"); } DWORD writtenWords; - WriteFile(pFile->hFileWrite, pData, WordCount * WordSize, &writtenWords, NULL); //FIXME: does not work for writing buffers larger than 2GB (even though this should rarely be the case in practice) + WriteFile(io.hWrite, pData, WordCount * WordSize, &writtenWords, NULL); //FIXME: does not work for writing buffers larger than 2GB (even though this should rarely be the case in practice) if (writtenWords < 1) throw Exception("Windows IO Error while trying to write chunk data"); writtenWords /= WordSize; #else // standard C functions - if (fseeko(pFile->hFileWrite, ullStartPos + ullPos, SEEK_SET)) { - throw Exception("Could not seek to position " + ToString(ullPos) + - " in chunk (" + ToString(ullStartPos + ullPos) + " in file)"); + if (fseeko(io.hWrite, ullStartPos + pos, SEEK_SET)) { + throw Exception("Could not seek to position " + ToString(pos) + + " in chunk (" + ToString(ullStartPos + pos) + " in file)"); } - file_offset_t writtenWords = fwrite(pData, WordSize, WordCount, pFile->hFileWrite); + file_offset_t writtenWords = fwrite(pData, WordSize, WordCount, io.hWrite); #endif // POSIX SetPos(writtenWords * WordSize, stream_curpos); return writtenWords; @@ -513,6 +588,7 @@ * @returns number of read integers * @throws RIFF::Exception if an error occurred or less than * \a WordCount integers could be read! + * @see File::IsIOPerThread() for multi-threaded streaming */ file_offset_t Chunk::ReadInt8(int8_t* pData, file_offset_t WordCount) { #if DEBUG_RIFF @@ -534,6 +610,7 @@ * @returns number of written integers * @throws RIFF::Exception if an IO error occurred * @see Resize() + * @see File::IsIOPerThread() for multi-threaded streaming */ file_offset_t Chunk::WriteInt8(int8_t* pData, file_offset_t WordCount) { return Write(pData, WordCount, 1); @@ -550,6 +627,7 @@ * @returns number of read integers * @throws RIFF::Exception if an error occurred or less than * \a WordCount integers could be read! + * @see File::IsIOPerThread() for multi-threaded streaming */ file_offset_t Chunk::ReadUint8(uint8_t* pData, file_offset_t WordCount) { #if DEBUG_RIFF @@ -571,6 +649,7 @@ * @returns number of written integers * @throws RIFF::Exception if an IO error occurred * @see Resize() + * @see File::IsIOPerThread() for multi-threaded streaming */ file_offset_t Chunk::WriteUint8(uint8_t* pData, file_offset_t WordCount) { return Write(pData, WordCount, 1); @@ -587,6 +666,7 @@ * @returns number of read integers * @throws RIFF::Exception if an error occurred or less than * \a WordCount integers could be read! + * @see File::IsIOPerThread() for multi-threaded streaming */ file_offset_t Chunk::ReadInt16(int16_t* pData, file_offset_t WordCount) { #if DEBUG_RIFF @@ -608,6 +688,7 @@ * @returns number of written integers * @throws RIFF::Exception if an IO error occurred * @see Resize() + * @see File::IsIOPerThread() for multi-threaded streaming */ file_offset_t Chunk::WriteInt16(int16_t* pData, file_offset_t WordCount) { return Write(pData, WordCount, 2); @@ -624,6 +705,7 @@ * @returns number of read integers * @throws RIFF::Exception if an error occurred or less than * \a WordCount integers could be read! + * @see File::IsIOPerThread() for multi-threaded streaming */ file_offset_t Chunk::ReadUint16(uint16_t* pData, file_offset_t WordCount) { #if DEBUG_RIFF @@ -645,6 +727,7 @@ * @returns number of written integers * @throws RIFF::Exception if an IO error occurred * @see Resize() + * @see File::IsIOPerThread() for multi-threaded streaming */ file_offset_t Chunk::WriteUint16(uint16_t* pData, file_offset_t WordCount) { return Write(pData, WordCount, 2); @@ -661,6 +744,7 @@ * @returns number of read integers * @throws RIFF::Exception if an error occurred or less than * \a WordCount integers could be read! + * @see File::IsIOPerThread() for multi-threaded streaming */ file_offset_t Chunk::ReadInt32(int32_t* pData, file_offset_t WordCount) { #if DEBUG_RIFF @@ -682,6 +766,7 @@ * @returns number of written integers * @throws RIFF::Exception if an IO error occurred * @see Resize() + * @see File::IsIOPerThread() for multi-threaded streaming */ file_offset_t Chunk::WriteInt32(int32_t* pData, file_offset_t WordCount) { return Write(pData, WordCount, 4); @@ -698,6 +783,7 @@ * @returns number of read integers * @throws RIFF::Exception if an error occurred or less than * \a WordCount integers could be read! + * @see File::IsIOPerThread() for multi-threaded streaming */ file_offset_t Chunk::ReadUint32(uint32_t* pData, file_offset_t WordCount) { #if DEBUG_RIFF @@ -715,6 +801,7 @@ * @param size number of characters to read * @throws RIFF::Exception if an error occurred or less than * \a size characters could be read! + * @see File::IsIOPerThread() for multi-threaded streaming */ void Chunk::ReadString(String& s, int size) { char* buf = new char[size]; @@ -736,6 +823,7 @@ * @returns number of written integers * @throws RIFF::Exception if an IO error occurred * @see Resize() + * @see File::IsIOPerThread() for multi-threaded streaming */ file_offset_t Chunk::WriteUint32(uint32_t* pData, file_offset_t WordCount) { return Write(pData, WordCount, 4); @@ -747,6 +835,7 @@ * * @returns read integer word * @throws RIFF::Exception if an error occurred + * @see File::IsIOPerThread() for multi-threaded streaming */ int8_t Chunk::ReadInt8() { #if DEBUG_RIFF @@ -763,6 +852,7 @@ * * @returns read integer word * @throws RIFF::Exception if an error occurred + * @see File::IsIOPerThread() for multi-threaded streaming */ uint8_t Chunk::ReadUint8() { #if DEBUG_RIFF @@ -780,6 +870,7 @@ * * @returns read integer word * @throws RIFF::Exception if an error occurred + * @see File::IsIOPerThread() for multi-threaded streaming */ int16_t Chunk::ReadInt16() { #if DEBUG_RIFF @@ -797,6 +888,7 @@ * * @returns read integer word * @throws RIFF::Exception if an error occurred + * @see File::IsIOPerThread() for multi-threaded streaming */ uint16_t Chunk::ReadUint16() { #if DEBUG_RIFF @@ -814,6 +906,7 @@ * * @returns read integer word * @throws RIFF::Exception if an error occurred + * @see File::IsIOPerThread() for multi-threaded streaming */ int32_t Chunk::ReadInt32() { #if DEBUG_RIFF @@ -831,6 +924,7 @@ * * @returns read integer word * @throws RIFF::Exception if an error occurred + * @see File::IsIOPerThread() for multi-threaded streamings */ uint32_t Chunk::ReadUint32() { #if DEBUG_RIFF @@ -861,29 +955,31 @@ * @returns a pointer to the data in RAM on success, NULL otherwise * @throws Exception if data buffer could not be enlarged * @see ReleaseChunkData() + * @see File::IsIOPerThread() for multi-threaded streaming */ void* Chunk::LoadChunkData() { if (!pChunkData && pFile->Filename != "" /*&& ulStartPos != 0*/) { + File::Handle hRead = pFile->FileHandle(); #if POSIX - if (lseek(pFile->hFileRead, ullStartPos, SEEK_SET) == -1) return NULL; + if (lseek(hRead, ullStartPos, SEEK_SET) == -1) return NULL; #elif defined(WIN32) LARGE_INTEGER liFilePos; liFilePos.QuadPart = ullStartPos; - if (!SetFilePointerEx(pFile->hFileRead, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN)) return NULL; + if (!SetFilePointerEx(hRead, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN)) return NULL; #else - if (fseeko(pFile->hFileRead, ullStartPos, SEEK_SET)) return NULL; + if (fseeko(hRead, ullStartPos, SEEK_SET)) return NULL; #endif // POSIX file_offset_t ullBufferSize = (ullCurrentChunkSize > ullNewChunkSize) ? ullCurrentChunkSize : ullNewChunkSize; pChunkData = new uint8_t[ullBufferSize]; if (!pChunkData) return NULL; memset(pChunkData, 0, ullBufferSize); #if POSIX - file_offset_t readWords = read(pFile->hFileRead, pChunkData, GetSize()); + file_offset_t readWords = read(hRead, pChunkData, GetSize()); #elif defined(WIN32) DWORD readWords; - ReadFile(pFile->hFileRead, pChunkData, GetSize(), &readWords, NULL); //FIXME: won't load chunks larger than 2GB ! + ReadFile(hRead, pChunkData, GetSize(), &readWords, NULL); //FIXME: won't load chunks larger than 2GB ! #else - file_offset_t readWords = fread(pChunkData, 1, GetSize(), pFile->hFileRead); + file_offset_t readWords = fread(pChunkData, 1, GetSize(), hRead); #endif // POSIX if (readWords != GetSize()) { delete[] pChunkData; @@ -894,8 +990,10 @@ uint8_t* pNewBuffer = new uint8_t[ullNewChunkSize]; if (!pNewBuffer) throw Exception("Could not enlarge chunk data buffer to " + ToString(ullNewChunkSize) + " bytes"); memset(pNewBuffer, 0 , ullNewChunkSize); - memcpy(pNewBuffer, pChunkData, ullChunkDataSize); - delete[] pChunkData; + if (pChunkData) { + memcpy(pNewBuffer, pChunkData, ullChunkDataSize); + delete[] pChunkData; + } pChunkData = pNewBuffer; ullChunkDataSize = ullNewChunkSize; } @@ -954,12 +1052,15 @@ * @returns new write position in the "physical" file, that is * \a ullWritePos incremented by this chunk's new size * (including its header size of course) + * @see File::IsIOPerThread() for multi-threaded streaming */ file_offset_t Chunk::WriteChunk(file_offset_t ullWritePos, file_offset_t ullCurrentDataOffset, progress_t* pProgress) { const file_offset_t ullOriginalPos = ullWritePos; ullWritePos += CHUNK_HEADER_SIZE(pFile->FileOffsetSize); - if (pFile->Mode != stream_mode_read_write) + const File::HandlePair io = pFile->FileHandlePair(); + + if (io.Mode != stream_mode_read_write) throw Exception("Cannot write list chunk, file has to be opened in read+write mode"); // if the whole chunk body was loaded into RAM @@ -968,22 +1069,22 @@ LoadChunkData(); // write chunk data from RAM persistently to the file #if POSIX - lseek(pFile->hFileWrite, ullWritePos, SEEK_SET); - if (write(pFile->hFileWrite, pChunkData, ullNewChunkSize) != ullNewChunkSize) { + lseek(io.hWrite, ullWritePos, SEEK_SET); + if (write(io.hWrite, pChunkData, ullNewChunkSize) != ullNewChunkSize) { throw Exception("Writing Chunk data (from RAM) failed"); } #elif defined(WIN32) LARGE_INTEGER liFilePos; liFilePos.QuadPart = ullWritePos; - SetFilePointerEx(pFile->hFileWrite, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN); + SetFilePointerEx(io.hWrite, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN); DWORD dwBytesWritten; - WriteFile(pFile->hFileWrite, pChunkData, ullNewChunkSize, &dwBytesWritten, NULL); //FIXME: won't save chunks larger than 2GB ! + WriteFile(io.hWrite, pChunkData, ullNewChunkSize, &dwBytesWritten, NULL); //FIXME: won't save chunks larger than 2GB ! if (dwBytesWritten != ullNewChunkSize) { throw Exception("Writing Chunk data (from RAM) failed"); } #else - fseeko(pFile->hFileWrite, ullWritePos, SEEK_SET); - if (fwrite(pChunkData, 1, ullNewChunkSize, pFile->hFileWrite) != ullNewChunkSize) { + fseeko(io.hWrite, ullWritePos, SEEK_SET); + if (fwrite(pChunkData, 1, ullNewChunkSize, io.hWrite) != ullNewChunkSize) { throw Exception("Writing Chunk data (from RAM) failed"); } #endif // POSIX @@ -999,23 +1100,23 @@ for (file_offset_t ullOffset = 0; ullToMove > 0 && iBytesMoved > 0; ullOffset += iBytesMoved, ullToMove -= iBytesMoved) { iBytesMoved = (ullToMove < 4096) ? int(ullToMove) : 4096; #if POSIX - lseek(pFile->hFileRead, ullStartPos + ullCurrentDataOffset + ullOffset, SEEK_SET); - iBytesMoved = (int) read(pFile->hFileRead, pCopyBuffer, (size_t) iBytesMoved); - lseek(pFile->hFileWrite, ullWritePos + ullOffset, SEEK_SET); - iBytesMoved = (int) write(pFile->hFileWrite, pCopyBuffer, (size_t) iBytesMoved); + lseek(io.hRead, ullStartPos + ullCurrentDataOffset + ullOffset, SEEK_SET); + iBytesMoved = (int) read(io.hRead, pCopyBuffer, (size_t) iBytesMoved); + lseek(io.hWrite, ullWritePos + ullOffset, SEEK_SET); + iBytesMoved = (int) write(io.hWrite, pCopyBuffer, (size_t) iBytesMoved); #elif defined(WIN32) LARGE_INTEGER liFilePos; liFilePos.QuadPart = ullStartPos + ullCurrentDataOffset + ullOffset; - SetFilePointerEx(pFile->hFileRead, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN); - ReadFile(pFile->hFileRead, pCopyBuffer, iBytesMoved, &iBytesMoved, NULL); + SetFilePointerEx(io.hRead, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN); + ReadFile(io.hRead, pCopyBuffer, iBytesMoved, &iBytesMoved, NULL); liFilePos.QuadPart = ullWritePos + ullOffset; - SetFilePointerEx(pFile->hFileWrite, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN); - WriteFile(pFile->hFileWrite, pCopyBuffer, iBytesMoved, &iBytesMoved, NULL); + SetFilePointerEx(io.hWrite, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN); + WriteFile(io.hWrite, pCopyBuffer, iBytesMoved, &iBytesMoved, NULL); #else - fseeko(pFile->hFileRead, ullStartPos + ullCurrentDataOffset + ullOffset, SEEK_SET); - iBytesMoved = fread(pCopyBuffer, 1, iBytesMoved, pFile->hFileRead); - fseeko(pFile->hFileWrite, ullWritePos + ullOffset, SEEK_SET); - iBytesMoved = fwrite(pCopyBuffer, 1, iBytesMoved, pFile->hFileWrite); + fseeko(io.hRead, ullStartPos + ullCurrentDataOffset + ullOffset, SEEK_SET); + iBytesMoved = fread(pCopyBuffer, 1, iBytesMoved, io.hRead); + fseeko(io.hWrite, ullWritePos + ullOffset, SEEK_SET); + iBytesMoved = fwrite(pCopyBuffer, 1, iBytesMoved, io.hWrite); #endif } delete[] pCopyBuffer; @@ -1026,27 +1127,28 @@ ullCurrentChunkSize = ullNewChunkSize; WriteHeader(ullOriginalPos); - __notify_progress(pProgress, 1.0); // notify done + if (pProgress) + __notify_progress(pProgress, 1.0); // notify done // update chunk's position pointers ullStartPos = ullOriginalPos + CHUNK_HEADER_SIZE(pFile->FileOffsetSize); - ullPos = 0; + Chunk::__resetPos(); // add pad byte if needed if ((ullStartPos + ullNewChunkSize) % 2 != 0) { const char cPadByte = 0; #if POSIX - lseek(pFile->hFileWrite, ullStartPos + ullNewChunkSize, SEEK_SET); - write(pFile->hFileWrite, &cPadByte, 1); + lseek(io.hWrite, ullStartPos + ullNewChunkSize, SEEK_SET); + write(io.hWrite, &cPadByte, 1); #elif defined(WIN32) LARGE_INTEGER liFilePos; liFilePos.QuadPart = ullStartPos + ullNewChunkSize; - SetFilePointerEx(pFile->hFileWrite, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN); + SetFilePointerEx(io.hWrite, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN); DWORD dwBytesWritten; - WriteFile(pFile->hFileWrite, &cPadByte, 1, &dwBytesWritten, NULL); + WriteFile(io.hWrite, &cPadByte, 1, &dwBytesWritten, NULL); #else - fseeko(pFile->hFileWrite, ullStartPos + ullNewChunkSize, SEEK_SET); - fwrite(&cPadByte, 1, 1, pFile->hFileWrite); + fseeko(io.hWrite, ullStartPos + ullNewChunkSize, SEEK_SET); + fwrite(&cPadByte, 1, 1, io.hWrite); #endif return ullStartPos + ullNewChunkSize + 1; } @@ -1055,7 +1157,9 @@ } void Chunk::__resetPos() { - ullPos = 0; + std::lock_guard lock(chunkPos.mutex); + chunkPos.ullPos = 0; + chunkPos.byThread.clear(); } @@ -1437,18 +1541,21 @@ Chunk::ReadHeader(filePos); if (ullCurrentChunkSize < 4) return; ullNewChunkSize = ullCurrentChunkSize -= 4; + + const File::Handle hRead = pFile->FileHandle(); + #if POSIX - lseek(pFile->hFileRead, filePos + CHUNK_HEADER_SIZE(pFile->FileOffsetSize), SEEK_SET); - read(pFile->hFileRead, &ListType, 4); + lseek(hRead, filePos + CHUNK_HEADER_SIZE(pFile->FileOffsetSize), SEEK_SET); + read(hRead, &ListType, 4); #elif defined(WIN32) LARGE_INTEGER liFilePos; liFilePos.QuadPart = filePos + CHUNK_HEADER_SIZE(pFile->FileOffsetSize); - SetFilePointerEx(pFile->hFileRead, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN); + SetFilePointerEx(hRead, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN); DWORD dwBytesRead; - ReadFile(pFile->hFileRead, &ListType, 4, &dwBytesRead, NULL); + ReadFile(hRead, &ListType, 4, &dwBytesRead, NULL); #else - fseeko(pFile->hFileRead, filePos + CHUNK_HEADER_SIZE(pFile->FileOffsetSize), SEEK_SET); - fread(&ListType, 4, 1, pFile->hFileRead); + fseeko(hRead, filePos + CHUNK_HEADER_SIZE(pFile->FileOffsetSize), SEEK_SET); + fread(&ListType, 4, 1, hRead); #endif // POSIX #if DEBUG_RIFF std::cout << "listType=" << convertToString(ListType) << std::endl; @@ -1463,18 +1570,21 @@ ullNewChunkSize += 4; Chunk::WriteHeader(filePos); ullNewChunkSize -= 4; // just revert the +4 incrementation + + const File::Handle hWrite = pFile->FileWriteHandle(); + #if POSIX - lseek(pFile->hFileWrite, filePos + CHUNK_HEADER_SIZE(pFile->FileOffsetSize), SEEK_SET); - write(pFile->hFileWrite, &ListType, 4); + lseek(hWrite, filePos + CHUNK_HEADER_SIZE(pFile->FileOffsetSize), SEEK_SET); + write(hWrite, &ListType, 4); #elif defined(WIN32) LARGE_INTEGER liFilePos; liFilePos.QuadPart = filePos + CHUNK_HEADER_SIZE(pFile->FileOffsetSize); - SetFilePointerEx(pFile->hFileWrite, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN); + SetFilePointerEx(hWrite, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN); DWORD dwBytesWritten; - WriteFile(pFile->hFileWrite, &ListType, 4, &dwBytesWritten, NULL); + WriteFile(hWrite, &ListType, 4, &dwBytesWritten, NULL); #else - fseeko(pFile->hFileWrite, filePos + CHUNK_HEADER_SIZE(pFile->FileOffsetSize), SEEK_SET); - fwrite(&ListType, 4, 1, pFile->hFileWrite); + fseeko(hWrite, filePos + CHUNK_HEADER_SIZE(pFile->FileOffsetSize), SEEK_SET); + fwrite(&ListType, 4, 1, hWrite); #endif // POSIX } @@ -1485,26 +1595,29 @@ if (!pSubChunks) { pSubChunks = new ChunkList(); pSubChunksMap = new ChunkMap(); - #if defined(WIN32) - if (pFile->hFileRead == INVALID_HANDLE_VALUE) return; - #else - if (!pFile->hFileRead) return; - #endif - file_offset_t ullOriginalPos = GetPos(); + + const File::Handle hRead = pFile->FileHandle(); + if (!_isValidHandle(hRead)) return; + + const file_offset_t ullOriginalPos = GetPos(); SetPos(0); // jump to beginning of list chunk body while (RemainingBytes() >= CHUNK_HEADER_SIZE(pFile->FileOffsetSize)) { Chunk* ck; uint32_t ckid; - Read(&ckid, 4, 1); + // return value check is required here to prevent a potential + // garbage data use of 'ckid' below in case Read() failed + if (Read(&ckid, 4, 1) != 4) + throw Exception("LoadSubChunks(): Failed reading RIFF chunk ID"); #if DEBUG_RIFF std::cout << " ckid=" << convertToString(ckid) << std::endl; #endif // DEBUG_RIFF + const file_offset_t pos = GetPos(); if (ckid == CHUNK_ID_LIST) { - ck = new RIFF::List(pFile, ullStartPos + ullPos - 4, this); + ck = new RIFF::List(pFile, ullStartPos + pos - 4, this); SetPos(ck->GetSize() + LIST_HEADER_SIZE(pFile->FileOffsetSize) - 4, RIFF::stream_curpos); } else { // simple chunk - ck = new RIFF::Chunk(pFile, ullStartPos + ullPos - 4, this); + ck = new RIFF::Chunk(pFile, ullStartPos + pos - 4, this); SetPos(ck->GetSize() + CHUNK_HEADER_SIZE(pFile->FileOffsetSize) - 4, RIFF::stream_curpos); } pSubChunks->push_back(ck); @@ -1513,20 +1626,25 @@ } SetPos(ullOriginalPos); // restore position before this call } - __notify_progress(pProgress, 1.0); // notify done + if (pProgress) + __notify_progress(pProgress, 1.0); // notify done } void List::LoadSubChunksRecursively(progress_t* pProgress) { const int n = (int) CountSubLists(); int i = 0; for (List* pList = GetFirstSubList(); pList; pList = GetNextSubList(), ++i) { - // divide local progress into subprogress - progress_t subprogress; - __divide_progress(pProgress, &subprogress, n, i); - // do the actual work - pList->LoadSubChunksRecursively(&subprogress); + if (pProgress) { + // divide local progress into subprogress + progress_t subprogress; + __divide_progress(pProgress, &subprogress, n, i); + // do the actual work + pList->LoadSubChunksRecursively(&subprogress); + } else + pList->LoadSubChunksRecursively(NULL); } - __notify_progress(pProgress, 1.0); // notify done + if (pProgress) + __notify_progress(pProgress, 1.0); // notify done } /** @brief Write list chunk persistently e.g. to disk. @@ -1548,7 +1666,7 @@ const file_offset_t ullOriginalPos = ullWritePos; ullWritePos += LIST_HEADER_SIZE(pFile->FileOffsetSize); - if (pFile->Mode != stream_mode_read_write) + if (pFile->GetMode() != stream_mode_read_write) throw Exception("Cannot write list chunk, file has to be opened in read+write mode"); // write all subchunks (including sub list chunks) recursively @@ -1556,11 +1674,14 @@ size_t i = 0; const size_t n = pSubChunks->size(); for (ChunkList::iterator iter = pSubChunks->begin(), end = pSubChunks->end(); iter != end; ++iter, ++i) { - // divide local progress into subprogress for loading current Instrument - progress_t subprogress; - __divide_progress(pProgress, &subprogress, n, i); - // do the actual work - ullWritePos = (*iter)->WriteChunk(ullWritePos, ullCurrentDataOffset, &subprogress); + if (pProgress) { + // divide local progress into subprogress for loading current Instrument + progress_t subprogress; + __divide_progress(pProgress, &subprogress, n, i); + // do the actual work + ullWritePos = (*iter)->WriteChunk(ullWritePos, ullCurrentDataOffset, &subprogress); + } else + ullWritePos = (*iter)->WriteChunk(ullWritePos, ullCurrentDataOffset, NULL); } } @@ -1571,7 +1692,8 @@ // offset of this list chunk in new written file may have changed ullStartPos = ullOriginalPos + LIST_HEADER_SIZE(pFile->FileOffsetSize); - __notify_progress(pProgress, 1.0); // notify done + if (pProgress) + __notify_progress(pProgress, 1.0); // notify done return ullWritePos; } @@ -1615,12 +1737,13 @@ : List(this), bIsNewFile(true), Layout(layout_standard), FileOffsetPreference(offset_size_auto) { + io.isPerThread = false; #if defined(WIN32) - hFileRead = hFileWrite = INVALID_HANDLE_VALUE; + io.hRead = io.hWrite = INVALID_HANDLE_VALUE; #else - hFileRead = hFileWrite = 0; + io.hRead = io.hWrite = 0; #endif - Mode = stream_mode_closed; + io.Mode = stream_mode_closed; bEndianNative = true; ListType = FileType; FileOffsetSize = 4; @@ -1710,30 +1833,31 @@ * given RIFF file or RIFF-alike file */ void File::__openExistingFile(const String& path, uint32_t* FileType) { + io.isPerThread = false; #if POSIX - hFileRead = hFileWrite = open(path.c_str(), O_RDONLY | O_NONBLOCK); - if (hFileRead == -1) { - hFileRead = hFileWrite = 0; + io.hRead = io.hWrite = open(path.c_str(), O_RDONLY | O_NONBLOCK); + if (io.hRead == -1) { + io.hRead = io.hWrite = 0; String sError = strerror(errno); throw RIFF::Exception("Can't open \"" + path + "\": " + sError); } #elif defined(WIN32) - hFileRead = hFileWrite = CreateFile( + io.hRead = io.hWrite = CreateFile( path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS, NULL ); - if (hFileRead == INVALID_HANDLE_VALUE) { - hFileRead = hFileWrite = INVALID_HANDLE_VALUE; + if (io.hRead == INVALID_HANDLE_VALUE) { + io.hRead = io.hWrite = INVALID_HANDLE_VALUE; throw RIFF::Exception("Can't open \"" + path + "\""); } #else - hFileRead = hFileWrite = fopen(path.c_str(), "rb"); - if (!hFileRead) throw RIFF::Exception("Can't open \"" + path + "\""); + io.hRead = io.hWrite = fopen(path.c_str(), "rb"); + if (!io.hRead) throw RIFF::Exception("Can't open \"" + path + "\""); #endif // POSIX - Mode = stream_mode_read; + io.Mode = stream_mode_read; // determine RIFF file offset size to be used (in RIFF chunk headers) // according to the current file offset preference @@ -1772,8 +1896,85 @@ Filename = path; } + /** + * This is an internal-only method which must not be used by any application + * and might change at any time. + * + * Resolves and returns a reference (memory location) of the RIFF file's + * internal and OS dependent file I/O handles which are intended to be used + * by the calling thread. + */ + File::HandlePair& File::FileHandlePairUnsafeRef() { + if (io.byThread.empty()) return io; + const std::thread::id tid = std::this_thread::get_id(); + const auto it = io.byThread.find(tid); + return (it != io.byThread.end()) ? + it->second : + io.byThread[tid] = { + #if defined(WIN32) + .hRead = INVALID_HANDLE_VALUE, + .hWrite = INVALID_HANDLE_VALUE, + #else + .hRead = 0, + .hWrite = 0, + #endif + .Mode = stream_mode_closed + }; + } + + /** + * Returns the OS dependent file I/O read and write handles intended to be + * used by the calling thread. + * + * @see File::IsIOPerThread() for multi-threaded streaming + */ + File::HandlePair File::FileHandlePair() const { + std::lock_guard lock(io.mutex); + if (io.byThread.empty()) return io; + const std::thread::id tid = std::this_thread::get_id(); + const auto it = io.byThread.find(tid); + return (it != io.byThread.end()) ? + it->second : + io.byThread[tid] = { + #if defined(WIN32) + .hRead = INVALID_HANDLE_VALUE, + .hWrite = INVALID_HANDLE_VALUE, + #else + .hRead = 0, + .hWrite = 0, + #endif + .Mode = stream_mode_closed + }; + } + + /** + * Returns the OS dependent file I/O read handle intended to be used by the + * calling thread. + * + * @see File::IsIOPerThread() for multi-threaded streaming + */ + File::Handle File::FileHandle() const { + return FileHandlePair().hRead; + } + + /** + * Returns the OS dependent file I/O write handle intended to be used by the + * calling thread. + * + * @see File::IsIOPerThread() for multi-threaded streaming + */ + File::Handle File::FileWriteHandle() const { + return FileHandlePair().hWrite; + } + + /** + * Returns the file I/O mode currently being available for the calling + * thread for this RIFF file (either ro, rw or closed). + * + * @see File::IsIOPerThread() for multi-threaded streaming + */ stream_mode_t File::GetMode() const { - return Mode; + return FileHandlePair().Mode; } layout_t File::GetLayout() const { @@ -1788,23 +1989,37 @@ * @param NewMode - new file access mode * @returns true if mode was changed, false if current mode already * equals new mode - * @throws RIFF::Exception if new file access mode is unknown + * @throws RIFF::Exception if file could not be opened in requested file + * access mode or if passed access mode is unknown + * @see File::IsIOPerThread() for multi-threaded streaming */ bool File::SetMode(stream_mode_t NewMode) { - if (NewMode != Mode) { + bool bResetPos = false; + bool res = SetModeInternal(NewMode, &bResetPos); + // resetting position must be handled outside above's call to avoid any + // potential dead lock, as SetModeInternal() acquires a lock and + // __resetPos() acquires a lock by itself (not a theoretical issue!) + if (bResetPos) + __resetPos(); // reset read/write position of ALL 'Chunk' objects + return res; + } + + bool File::SetModeInternal(stream_mode_t NewMode, bool* pResetPos) { + std::lock_guard lock(io.mutex); + HandlePair& io = FileHandlePairUnsafeRef(); + if (NewMode != io.Mode) { switch (NewMode) { case stream_mode_read: + if (_isValidHandle(io.hRead)) _close(io.hRead); #if POSIX - if (hFileRead) close(hFileRead); - hFileRead = hFileWrite = open(Filename.c_str(), O_RDONLY | O_NONBLOCK); - if (hFileRead == -1) { - hFileRead = hFileWrite = 0; + io.hRead = io.hWrite = open(Filename.c_str(), O_RDONLY | O_NONBLOCK); + if (io.hRead == -1) { + io.hRead = io.hWrite = 0; String sError = strerror(errno); throw Exception("Could not (re)open file \"" + Filename + "\" in read mode: " + sError); } #elif defined(WIN32) - if (hFileRead != INVALID_HANDLE_VALUE) CloseHandle(hFileRead); - hFileRead = hFileWrite = CreateFile( + io.hRead = io.hWrite = CreateFile( Filename.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, @@ -1812,29 +2027,27 @@ FILE_FLAG_RANDOM_ACCESS, NULL ); - if (hFileRead == INVALID_HANDLE_VALUE) { - hFileRead = hFileWrite = INVALID_HANDLE_VALUE; + if (io.hRead == INVALID_HANDLE_VALUE) { + io.hRead = io.hWrite = INVALID_HANDLE_VALUE; throw Exception("Could not (re)open file \"" + Filename + "\" in read mode"); } #else - if (hFileRead) fclose(hFileRead); - hFileRead = hFileWrite = fopen(Filename.c_str(), "rb"); - if (!hFileRead) throw Exception("Could not (re)open file \"" + Filename + "\" in read mode"); + io.hRead = io.hWrite = fopen(Filename.c_str(), "rb"); + if (!io.hRead) throw Exception("Could not (re)open file \"" + Filename + "\" in read mode"); #endif - __resetPos(); // reset read/write position of ALL 'Chunk' objects + *pResetPos = true; break; case stream_mode_read_write: + if (_isValidHandle(io.hRead)) _close(io.hRead); #if POSIX - if (hFileRead) close(hFileRead); - hFileRead = hFileWrite = open(Filename.c_str(), O_RDWR | O_NONBLOCK); - if (hFileRead == -1) { - hFileRead = hFileWrite = open(Filename.c_str(), O_RDONLY | O_NONBLOCK); + io.hRead = io.hWrite = open(Filename.c_str(), O_RDWR | O_NONBLOCK); + if (io.hRead == -1) { + io.hRead = io.hWrite = open(Filename.c_str(), O_RDONLY | O_NONBLOCK); String sError = strerror(errno); throw Exception("Could not open file \"" + Filename + "\" in read+write mode: " + sError); } #elif defined(WIN32) - if (hFileRead != INVALID_HANDLE_VALUE) CloseHandle(hFileRead); - hFileRead = hFileWrite = CreateFile( + io.hRead = io.hWrite = CreateFile( Filename.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, @@ -1843,8 +2056,8 @@ FILE_FLAG_RANDOM_ACCESS, NULL ); - if (hFileRead == INVALID_HANDLE_VALUE) { - hFileRead = hFileWrite = CreateFile( + if (io.hRead == INVALID_HANDLE_VALUE) { + io.hRead = io.hWrite = CreateFile( Filename.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, @@ -1855,34 +2068,29 @@ throw Exception("Could not (re)open file \"" + Filename + "\" in read+write mode"); } #else - if (hFileRead) fclose(hFileRead); - hFileRead = hFileWrite = fopen(Filename.c_str(), "r+b"); - if (!hFileRead) { - hFileRead = hFileWrite = fopen(Filename.c_str(), "rb"); + io.hRead = io.hWrite = fopen(Filename.c_str(), "r+b"); + if (!io.hRead) { + io.hRead = io.hWrite = fopen(Filename.c_str(), "rb"); throw Exception("Could not open file \"" + Filename + "\" in read+write mode"); } #endif - __resetPos(); // reset read/write position of ALL 'Chunk' objects + *pResetPos = true; break; case stream_mode_closed: + if (_isValidHandle(io.hRead)) _close(io.hRead); + if (_isValidHandle(io.hWrite)) _close(io.hWrite); #if POSIX - if (hFileRead) close(hFileRead); - if (hFileWrite) close(hFileWrite); - hFileRead = hFileWrite = 0; + io.hRead = io.hWrite = 0; #elif defined(WIN32) - if (hFileRead != INVALID_HANDLE_VALUE) CloseHandle(hFileRead); - if (hFileWrite != INVALID_HANDLE_VALUE) CloseHandle(hFileWrite); - hFileRead = hFileWrite = INVALID_HANDLE_VALUE; + io.hRead = io.hWrite = INVALID_HANDLE_VALUE; #else - if (hFileRead) fclose(hFileRead); - if (hFileWrite) fclose(hFileWrite); - hFileRead = hFileWrite = NULL; + io.hRead = io.hWrite = NULL; #endif break; default: throw Exception("Unknown file access mode"); } - Mode = NewMode; + io.Mode = NewMode; return true; } return false; @@ -1913,6 +2121,7 @@ * @param pProgress - optional: callback function for progress notification * @throws RIFF::Exception if there is an empty chunk or empty list * chunk or any kind of IO error occurred + * @see File::IsIOPerThread() for multi-threaded streaming */ void File::Save(progress_t* pProgress) { //TODO: implementation for the case where first chunk is not a global container (List chunk) is not implemented yet (i.e. Korg files) @@ -1920,7 +2129,7 @@ throw Exception("Saving a RIFF file with layout_flat is not implemented yet"); // make sure the RIFF tree is built (from the original file) - { + if (pProgress) { // divide progress into subprogress progress_t subprogress; __divide_progress(pProgress, &subprogress, 3.f, 0.f); // arbitrarily subdivided into 1/3 of total progress @@ -1928,7 +2137,8 @@ LoadSubChunksRecursively(&subprogress); // notify subprogress done __notify_progress(&subprogress, 1.f); - } + } else + LoadSubChunksRecursively(NULL); // reopen file in write mode SetMode(stream_mode_read_write); @@ -1943,6 +2153,10 @@ // the RIFF file offset size to be used accordingly for all chunks FileOffsetSize = FileOffsetSizeFor(newFileSize); + const HandlePair io = FileHandlePair(); + const Handle hRead = io.hRead; + const Handle hWrite = io.hWrite; + // to be able to save the whole file without loading everything into // RAM and without having to store the data in a temporary file, we // enlarge the file with the overall positive file size change, @@ -1958,7 +2172,8 @@ // divide progress into subprogress progress_t subprogress; - __divide_progress(pProgress, &subprogress, 3.f, 1.f); // arbitrarily subdivided into 1/3 of total progress + if (pProgress) + __divide_progress(pProgress, &subprogress, 3.f, 1.f); // arbitrarily subdivided into 1/3 of total progress // ... we enlarge this file first ... ResizeFile(newFileSize); @@ -1974,48 +2189,52 @@ iBytesMoved = (ullPos < 4096) ? ullPos : 4096; ullPos -= iBytesMoved; #if POSIX - lseek(hFileRead, ullPos, SEEK_SET); - iBytesMoved = read(hFileRead, pCopyBuffer, iBytesMoved); - lseek(hFileWrite, ullPos + positiveSizeDiff, SEEK_SET); - iBytesMoved = write(hFileWrite, pCopyBuffer, iBytesMoved); + lseek(hRead, ullPos, SEEK_SET); + iBytesMoved = read(hRead, pCopyBuffer, iBytesMoved); + lseek(hWrite, ullPos + positiveSizeDiff, SEEK_SET); + iBytesMoved = write(hWrite, pCopyBuffer, iBytesMoved); #elif defined(WIN32) LARGE_INTEGER liFilePos; liFilePos.QuadPart = ullPos; - SetFilePointerEx(hFileRead, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN); - ReadFile(hFileRead, pCopyBuffer, iBytesMoved, &iBytesMoved, NULL); + SetFilePointerEx(hRead, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN); + ReadFile(hRead, pCopyBuffer, iBytesMoved, &iBytesMoved, NULL); liFilePos.QuadPart = ullPos + positiveSizeDiff; - SetFilePointerEx(hFileWrite, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN); - WriteFile(hFileWrite, pCopyBuffer, iBytesMoved, &iBytesMoved, NULL); + SetFilePointerEx(hWrite, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN); + WriteFile(hWrite, pCopyBuffer, iBytesMoved, &iBytesMoved, NULL); #else - fseeko(hFileRead, ullPos, SEEK_SET); - iBytesMoved = fread(pCopyBuffer, 1, iBytesMoved, hFileRead); - fseeko(hFileWrite, ullPos + positiveSizeDiff, SEEK_SET); - iBytesMoved = fwrite(pCopyBuffer, 1, iBytesMoved, hFileWrite); + fseeko(hRead, ullPos, SEEK_SET); + iBytesMoved = fread(pCopyBuffer, 1, iBytesMoved, hRead); + fseeko(hWrite, ullPos + positiveSizeDiff, SEEK_SET); + iBytesMoved = fwrite(pCopyBuffer, 1, iBytesMoved, hWrite); #endif - if (!(iNotif % 8) && iBytesMoved > 0) + if (pProgress && !(iNotif % 8) && iBytesMoved > 0) __notify_progress(&subprogress, float(workingFileSize - ullPos) / float(workingFileSize)); } delete[] pCopyBuffer; if (iBytesMoved < 0) throw Exception("Could not modify file while trying to enlarge it"); - __notify_progress(&subprogress, 1.f); // notify subprogress done + if (pProgress) + __notify_progress(&subprogress, 1.f); // notify subprogress done } // rebuild / rewrite complete RIFF tree ... // divide progress into subprogress progress_t subprogress; - __divide_progress(pProgress, &subprogress, 3.f, 2.f); // arbitrarily subdivided into 1/3 of total progress + if (pProgress) + __divide_progress(pProgress, &subprogress, 3.f, 2.f); // arbitrarily subdivided into 1/3 of total progress // do the actual work - const file_offset_t finalSize = WriteChunk(0, positiveSizeDiff, &subprogress); - const file_offset_t finalActualSize = __GetFileSize(hFileWrite); + const file_offset_t finalSize = WriteChunk(0, positiveSizeDiff, pProgress ? &subprogress : NULL); + const file_offset_t finalActualSize = __GetFileSize(hWrite); // notify subprogress done - __notify_progress(&subprogress, 1.f); + if (pProgress) + __notify_progress(&subprogress, 1.f); // resize file to the final size if (finalSize < finalActualSize) ResizeFile(finalSize); - __notify_progress(pProgress, 1.0); // notify done + if (pProgress) + __notify_progress(pProgress, 1.0); // notify done } /** @brief Save changes to another file. @@ -2031,6 +2250,7 @@ * * @param path - path and file name where everything should be written to * @param pProgress - optional: callback function for progress notification + * @see File::IsIOPerThread() for multi-threaded streaming */ void File::Save(const String& path, progress_t* pProgress) { //TODO: we should make a check here if somebody tries to write to the same file and automatically call the other Save() method in that case @@ -2040,7 +2260,7 @@ throw Exception("Saving a RIFF file with layout_flat is not implemented yet"); // make sure the RIFF tree is built (from the original file) - { + if (pProgress) { // divide progress into subprogress progress_t subprogress; __divide_progress(pProgress, &subprogress, 2.f, 0.f); // arbitrarily subdivided into 1/2 of total progress @@ -2048,35 +2268,42 @@ LoadSubChunksRecursively(&subprogress); // notify subprogress done __notify_progress(&subprogress, 1.f); - } + } else + LoadSubChunksRecursively(NULL); if (!bIsNewFile) SetMode(stream_mode_read); - // open the other (new) file for writing and truncate it to zero size - #if POSIX - hFileWrite = open(path.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP); - if (hFileWrite == -1) { - hFileWrite = hFileRead; - String sError = strerror(errno); - throw Exception("Could not open file \"" + path + "\" for writing: " + sError); - } - #elif defined(WIN32) - hFileWrite = CreateFile( - path.c_str(), GENERIC_WRITE, FILE_SHARE_READ, - NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL | - FILE_FLAG_RANDOM_ACCESS, NULL - ); - if (hFileWrite == INVALID_HANDLE_VALUE) { - hFileWrite = hFileRead; - throw Exception("Could not open file \"" + path + "\" for writing"); - } - #else - hFileWrite = fopen(path.c_str(), "w+b"); - if (!hFileWrite) { - hFileWrite = hFileRead; - throw Exception("Could not open file \"" + path + "\" for writing"); + + { + std::lock_guard lock(io.mutex); + HandlePair& io = FileHandlePairUnsafeRef(); + + // open the other (new) file for writing and truncate it to zero size + #if POSIX + io.hWrite = open(path.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP); + if (io.hWrite == -1) { + io.hWrite = io.hRead; + String sError = strerror(errno); + throw Exception("Could not open file \"" + path + "\" for writing: " + sError); + } + #elif defined(WIN32) + io.hWrite = CreateFile( + path.c_str(), GENERIC_WRITE, FILE_SHARE_READ, + NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL | + FILE_FLAG_RANDOM_ACCESS, NULL + ); + if (io.hWrite == INVALID_HANDLE_VALUE) { + io.hWrite = io.hRead; + throw Exception("Could not open file \"" + path + "\" for writing"); + } + #else + io.hWrite = fopen(path.c_str(), "w+b"); + if (!io.hWrite) { + io.hWrite = io.hRead; + throw Exception("Could not open file \"" + path + "\" for writing"); + } + #endif // POSIX + io.Mode = stream_mode_read_write; } - #endif // POSIX - Mode = stream_mode_read_write; // get the overall file size required to save this file const file_offset_t newFileSize = GetRequiredFileSize(FileOffsetPreference); @@ -2087,7 +2314,7 @@ // write complete RIFF tree to the other (new) file file_offset_t ullTotalSize; - { + if (pProgress) { // divide progress into subprogress progress_t subprogress; __divide_progress(pProgress, &subprogress, 2.f, 1.f); // arbitrarily subdivided into 1/2 of total progress @@ -2095,40 +2322,43 @@ ullTotalSize = WriteChunk(0, 0, &subprogress); // notify subprogress done __notify_progress(&subprogress, 1.f); - } - file_offset_t ullActualSize = __GetFileSize(hFileWrite); + } else + ullTotalSize = WriteChunk(0, 0, NULL); + + const file_offset_t ullActualSize = __GetFileSize(FileWriteHandle()); // resize file to the final size (if the file was originally larger) if (ullActualSize > ullTotalSize) ResizeFile(ullTotalSize); - #if POSIX - if (hFileWrite) close(hFileWrite); - #elif defined(WIN32) - if (hFileWrite != INVALID_HANDLE_VALUE) CloseHandle(hFileWrite); - #else - if (hFileWrite) fclose(hFileWrite); - #endif - hFileWrite = hFileRead; + { + std::lock_guard lock(io.mutex); + HandlePair& io = FileHandlePairUnsafeRef(); - // associate new file with this File object from now on - Filename = path; - bIsNewFile = false; - Mode = (stream_mode_t) -1; // Just set it to an undefined mode ... + if (_isValidHandle(io.hWrite)) _close(io.hWrite); + io.hWrite = io.hRead; + + // associate new file with this File object from now on + Filename = path; + bIsNewFile = false; + io.Mode = (stream_mode_t) -1; // Just set it to an undefined mode ... + } SetMode(stream_mode_read_write); // ... so SetMode() has to reopen the file handles. - __notify_progress(pProgress, 1.0); // notify done + if (pProgress) + __notify_progress(pProgress, 1.0); // notify done } void File::ResizeFile(file_offset_t ullNewSize) { + const Handle hWrite = FileWriteHandle(); #if POSIX - if (ftruncate(hFileWrite, ullNewSize) < 0) + if (ftruncate(hWrite, ullNewSize) < 0) throw Exception("Could not resize file \"" + Filename + "\""); #elif defined(WIN32) LARGE_INTEGER liFilePos; liFilePos.QuadPart = ullNewSize; if ( - !SetFilePointerEx(hFileWrite, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN) || - !SetEndOfFile(hFileWrite) + !SetFilePointerEx(hWrite, liFilePos, NULL/*new pos pointer*/, FILE_BEGIN) || + !SetEndOfFile(hWrite) ) throw Exception("Could not resize file \"" + Filename + "\""); #else # error Sorry, this version of libgig only supports POSIX and Windows systems yet. @@ -2152,13 +2382,13 @@ } void File::Cleanup() { - #if POSIX - if (hFileRead) close(hFileRead); - #elif defined(WIN32) - if (hFileRead != INVALID_HANDLE_VALUE) CloseHandle(hFileRead); - #else - if (hFileRead) fclose(hFileRead); - #endif // POSIX + if (IsIOPerThread()) { + for (auto it = io.byThread.begin(); it != io.byThread.end(); ++it) { + _close(it->second.hRead); + } + } else { + _close(io.hRead); + } DeleteChunkList(); pFile = NULL; } @@ -2171,8 +2401,9 @@ */ file_offset_t File::GetCurrentFileSize() const { file_offset_t size = 0; + const Handle hRead = FileHandle(); try { - size = __GetFileSize(hFileRead); + size = __GetFileSize(hRead); } catch (...) { size = 0; } @@ -2280,6 +2511,83 @@ return FileOffsetSizeFor(GetCurrentFileSize()); } + /** @brief Whether file streams are independent for each thread. + * + * All file I/O operations like reading from a RIFF chunk body (e.g. by + * calling Chunk::Read(), Chunk::ReadInt8()), writing to a RIFF chunk body + * (e.g. by calling Chunk::Write(), Chunk::WriteInt8()) or saving the + * current RIFF tree structure to some file (e.g. by calling Save()) + * operate on a file I/O stream state, i.e. there is a "current" file + * read/write position and reading/writing by a certain amount of bytes + * automatically advances that "current" file position. + * + * By default there is only one stream state for a RIFF::File object, which + * is not an issue as long as only one thread is using the RIFF::File + * object at a time (which might also be the case in a collaborative / + * coroutine multi-threaded scenario). + * + * If however a RIFF::File object is read/written @b simultaniously by + * multiple threads this can lead to undefined behaviour as the individual + * threads would concurrently alter the file stream position. For such a + * concurrent multithreaded file I/O scenario @c SetIOPerThread(true) might + * be enabled which causes each thread to automatically use its own file + * stream state. + * + * @returns true if each thread has its own file stream state + * (default: false) + * @see SetIOPerThread() + */ + bool File::IsIOPerThread() const { + //NOTE: Not caring about atomicity here at all, for three reasons: + // 1. SetIOPerThread() is assumed to be called only once for the entire + // life time of a RIFF::File, usually very early at its lifetime, and + // hence a change to isPerThread should already safely be propagated + // before any other thread would actually read this boolean flag. + // 2. This method is called very frequently, and therefore any + // synchronization techique would hurt runtime efficiency. + // 3. Using even a mutex lock here might easily cause a deadlock due to + // other locks been taken in this .cpp file, i.e. at a higher call + // stack level (and this is the main reason why I removed it here). + return io.isPerThread; + } + + /** @brief Enable/disable file streams being independent for each thread. + * + * By enabling this feature (default off) each thread will automatically use + * its own file I/O stream state for allowing simultanious multi-threaded + * file read/write operations. + * + * @b NOTE: After having enabled this feature, the individual threads must + * at least once check GetState() and if their file I/O stream is yet closed + * they must call SetMode() (i.e. once) respectively to open their own file + * handles before being able to use any of the Read() or Write() methods. + * + * @param enable - @c true: one independent stream state per thread, + * @c false: only one stream in total shared by @b all threads + * @see IsIOPerThread() for more details about this feature + */ + void File::SetIOPerThread(bool enable) { + std::lock_guard lock(io.mutex); + if (!io.byThread.empty() == enable) return; + io.isPerThread = enable; + if (enable) { + const std::thread::id tid = std::this_thread::get_id(); + io.byThread[tid] = io; + } else { + // retain an arbitrary handle pair, close all other handle pairs + for (auto it = io.byThread.begin(); it != io.byThread.end(); ++it) { + if (it == io.byThread.begin()) { + io.hRead = it->second.hRead; + io.hWrite = it->second.hWrite; + } else { + _close(it->second.hRead); + _close(it->second.hWrite); + } + } + io.byThread.clear(); + } + } + #if POSIX file_offset_t File::__GetFileSize(int hFile) const { struct stat filestat;