--- libgig/trunk/src/gig.cpp 2014/06/07 22:28:04 2602 +++ libgig/trunk/src/gig.cpp 2016/09/20 22:13:37 2985 @@ -2,7 +2,7 @@ * * * libgig - C++ cross-platform Gigasampler format file access library * * * - * Copyright (C) 2003-2014 by Christian Schoenebeck * + * Copyright (C) 2003-2016 by Christian Schoenebeck * * * * * * This library is free software; you can redistribute it and/or modify * @@ -30,6 +30,10 @@ #include #include +/// libgig's current file format version (for extending the original Giga file +/// format with libgig's own custom data / custom features). +#define GIG_FILE_EXT_VERSION 2 + /// Initial size of the sample buffer which is used for decompression of /// compressed sample wave streams - this value should always be bigger than /// the biggest sample piece expected to be read by the sampler engine, @@ -53,38 +57,6 @@ namespace gig { -// *************** progress_t *************** -// * - - progress_t::progress_t() { - callback = NULL; - custom = NULL; - __range_min = 0.0f; - __range_max = 1.0f; - } - - // private helper function to convert progress of a subprocess into the global progress - static void __notify_progress(progress_t* pProgress, float subprogress) { - if (pProgress && pProgress->callback) { - const float totalrange = pProgress->__range_max - pProgress->__range_min; - const float totalprogress = pProgress->__range_min + subprogress * totalrange; - pProgress->factor = totalprogress; - pProgress->callback(pProgress); // now actually notify about the progress - } - } - - // private helper function to divide a progress into subprogresses - static void __divide_progress(progress_t* pParentProgress, progress_t* pSubProgress, float totalTasks, float currentTask) { - if (pParentProgress && pParentProgress->callback) { - const float totalrange = pParentProgress->__range_max - pParentProgress->__range_min; - pSubProgress->callback = pParentProgress->callback; - pSubProgress->custom = pParentProgress->custom; - pSubProgress->__range_min = pParentProgress->__range_min + totalrange * currentTask / totalTasks; - pSubProgress->__range_max = pSubProgress->__range_min + totalrange / totalTasks; - } - } - - // *************** Internal functions for sample decompression *************** // * @@ -123,8 +95,8 @@ void Decompress16(int compressionmode, const unsigned char* params, int srcStep, int dstStep, const unsigned char* pSrc, int16_t* pDst, - unsigned long currentframeoffset, - unsigned long copysamples) + file_offset_t currentframeoffset, + file_offset_t copysamples) { switch (compressionmode) { case 0: // 16 bit uncompressed @@ -160,8 +132,8 @@ void Decompress24(int compressionmode, const unsigned char* params, int dstStep, const unsigned char* pSrc, uint8_t* pDst, - unsigned long currentframeoffset, - unsigned long copysamples, int truncatedBits) + file_offset_t currentframeoffset, + file_offset_t copysamples, int truncatedBits) { int y, dy, ddy, dddy; @@ -346,7 +318,7 @@ // *************** Sample *************** // * - unsigned int Sample::Instances = 0; + size_t Sample::Instances = 0; buffer_t Sample::InternalDecompressionBuffer; /** @brief Constructor. @@ -367,7 +339,7 @@ * @param fileNo - number of an extension file where this sample * is located, 0 otherwise */ - Sample::Sample(File* pFile, RIFF::List* waveList, unsigned long WavePoolOffset, unsigned long fileNo) : DLS::Sample((DLS::File*) pFile, waveList, WavePoolOffset) { + Sample::Sample(File* pFile, RIFF::List* waveList, file_offset_t WavePoolOffset, unsigned long fileNo) : DLS::Sample((DLS::File*) pFile, waveList, WavePoolOffset) { static const DLS::Info::string_length_t fixedStringLengths[] = { { CHUNK_ID_INAM, 64 }, { 0, 0 } @@ -509,10 +481,10 @@ const int iReadAtOnce = 32*1024; char* buf = new char[iReadAtOnce * orig->FrameSize]; Sample* pOrig = (Sample*) orig; //HACK: remove constness for now - unsigned long restorePos = pOrig->GetPos(); + file_offset_t restorePos = pOrig->GetPos(); pOrig->SetPos(0); SetPos(0); - for (unsigned long n = pOrig->Read(buf, iReadAtOnce); n; + for (file_offset_t n = pOrig->Read(buf, iReadAtOnce); n; n = pOrig->Read(buf, iReadAtOnce)) { Write(buf, n); @@ -528,13 +500,14 @@ * Usually there is absolutely no need to call this method explicitly. * It will be called automatically when File::Save() was called. * + * @param pProgress - callback function for progress notification * @throws DLS::Exception if FormatTag != DLS_WAVE_FORMAT_PCM or no sample data * was provided yet * @throws gig::Exception if there is any invalid sample setting */ - void Sample::UpdateChunks() { + void Sample::UpdateChunks(progress_t* pProgress) { // first update base class's chunks - DLS::Sample::UpdateChunks(); + DLS::Sample::UpdateChunks(pProgress); // make sure 'smpl' chunk exists pCkSmpl = pWaveList->GetSubChunk(CHUNK_ID_SMPL); @@ -596,7 +569,7 @@ void Sample::ScanCompressedSample() { //TODO: we have to add some more scans here (e.g. determine compression rate) this->SamplesTotal = 0; - std::list frameOffsets; + std::list frameOffsets; SamplesPerFrame = BitDepth == 24 ? 256 : 2048; WorstCaseFrameSize = SamplesPerFrame * FrameSize + Channels; // +Channels for compression flag @@ -612,7 +585,7 @@ const int mode_l = pCkData->ReadUint8(); const int mode_r = pCkData->ReadUint8(); if (mode_l > 5 || mode_r > 5) throw gig::Exception("Unknown compression mode"); - const unsigned long frameSize = bytesPerFrame[mode_l] + bytesPerFrame[mode_r]; + const file_offset_t frameSize = bytesPerFrame[mode_l] + bytesPerFrame[mode_r]; if (pCkData->RemainingBytes() <= frameSize) { SamplesInLastFrame = @@ -631,7 +604,7 @@ const int mode = pCkData->ReadUint8(); if (mode > 5) throw gig::Exception("Unknown compression mode"); - const unsigned long frameSize = bytesPerFrame[mode]; + const file_offset_t frameSize = bytesPerFrame[mode]; if (pCkData->RemainingBytes() <= frameSize) { SamplesInLastFrame = @@ -647,9 +620,9 @@ // Build the frames table (which is used for fast resolving of a frame's chunk offset) if (FrameTable) delete[] FrameTable; - FrameTable = new unsigned long[frameOffsets.size()]; - std::list::iterator end = frameOffsets.end(); - std::list::iterator iter = frameOffsets.begin(); + FrameTable = new file_offset_t[frameOffsets.size()]; + std::list::iterator end = frameOffsets.end(); + std::list::iterator iter = frameOffsets.begin(); for (int i = 0; iter != end; i++, iter++) { FrameTable[i] = *iter; } @@ -690,7 +663,7 @@ * the cached sample data in bytes * @see ReleaseSampleData(), Read(), SetPos() */ - buffer_t Sample::LoadSampleData(unsigned long SampleCount) { + buffer_t Sample::LoadSampleData(file_offset_t SampleCount) { return LoadSampleDataWithNullSamplesExtension(SampleCount, 0); // 0 amount of NullSamples } @@ -749,10 +722,10 @@ * size of the cached sample data in bytes * @see ReleaseSampleData(), Read(), SetPos() */ - buffer_t Sample::LoadSampleDataWithNullSamplesExtension(unsigned long SampleCount, uint NullSamplesCount) { + buffer_t Sample::LoadSampleDataWithNullSamplesExtension(file_offset_t SampleCount, uint NullSamplesCount) { if (SampleCount > this->SamplesTotal) SampleCount = this->SamplesTotal; if (RAMCache.pStart) delete[] (int8_t*) RAMCache.pStart; - unsigned long allocationsize = (SampleCount + NullSamplesCount) * this->FrameSize; + file_offset_t allocationsize = (SampleCount + NullSamplesCount) * this->FrameSize; SetPos(0); // reset read position to begin of sample RAMCache.pStart = new int8_t[allocationsize]; RAMCache.Size = Read(RAMCache.pStart, SampleCount) * this->FrameSize; @@ -816,17 +789,17 @@ * FormatTag must be DLS_WAVE_FORMAT_PCM. Trying to resize samples with * other formats will fail! * - * @param iNewSize - new sample wave data size in sample points (must be - * greater than zero) + * @param NewSize - new sample wave data size in sample points (must be + * greater than zero) * @throws DLS::Excecption if FormatTag != DLS_WAVE_FORMAT_PCM - * or if \a iNewSize is less than 1 + * @throws DLS::Exception if \a NewSize is less than 1 or unrealistic large * @throws gig::Exception if existing sample is compressed * @see DLS::Sample::GetSize(), DLS::Sample::FrameSize, * DLS::Sample::FormatTag, File::Save() */ - void Sample::Resize(int iNewSize) { + void Sample::Resize(file_offset_t NewSize) { if (Compressed) throw gig::Exception("There is no support for modifying compressed samples (yet)"); - DLS::Sample::Resize(iNewSize); + DLS::Sample::Resize(NewSize); } /** @@ -850,7 +823,7 @@ * @returns the new sample position * @see Read() */ - unsigned long Sample::SetPos(unsigned long SampleCount, RIFF::stream_whence_t Whence) { + file_offset_t Sample::SetPos(file_offset_t SampleCount, RIFF::stream_whence_t Whence) { if (Compressed) { switch (Whence) { case RIFF::stream_curpos: @@ -868,14 +841,14 @@ } if (this->SamplePos > this->SamplesTotal) this->SamplePos = this->SamplesTotal; - unsigned long frame = this->SamplePos / 2048; // to which frame to jump + file_offset_t frame = this->SamplePos / 2048; // to which frame to jump this->FrameOffset = this->SamplePos % 2048; // offset (in sample points) within that frame pCkData->SetPos(FrameTable[frame]); // set chunk pointer to the start of sought frame return this->SamplePos; } else { // not compressed - unsigned long orderedBytes = SampleCount * this->FrameSize; - unsigned long result = pCkData->SetPos(orderedBytes, Whence); + file_offset_t orderedBytes = SampleCount * this->FrameSize; + file_offset_t result = pCkData->SetPos(orderedBytes, Whence); return (result == orderedBytes) ? SampleCount : result / this->FrameSize; } @@ -884,7 +857,7 @@ /** * Returns the current position in the sample (in sample points). */ - unsigned long Sample::GetPos() const { + file_offset_t Sample::GetPos() const { if (Compressed) return SamplePos; else return pCkData->GetPos() / FrameSize; } @@ -923,9 +896,9 @@ * @returns number of successfully read sample points * @see CreateDecompressionBuffer() */ - unsigned long Sample::ReadAndLoop(void* pBuffer, unsigned long SampleCount, playback_state_t* pPlaybackState, + file_offset_t Sample::ReadAndLoop(void* pBuffer, file_offset_t SampleCount, playback_state_t* pPlaybackState, DimensionRegion* pDimRgn, buffer_t* pExternalDecompressionBuffer) { - unsigned long samplestoread = SampleCount, totalreadsamples = 0, readsamples, samplestoloopend; + file_offset_t samplestoread = SampleCount, totalreadsamples = 0, readsamples, samplestoloopend; uint8_t* pDst = (uint8_t*) pBuffer; SetPos(pPlaybackState->position); // recover position from the last time @@ -963,10 +936,10 @@ // reading, swap all sample frames so it reflects // backward playback - unsigned long swapareastart = totalreadsamples; - unsigned long loopoffset = GetPos() - loop.LoopStart; - unsigned long samplestoreadinloop = Min(samplestoread, loopoffset); - unsigned long reverseplaybackend = GetPos() - samplestoreadinloop; + file_offset_t swapareastart = totalreadsamples; + file_offset_t loopoffset = GetPos() - loop.LoopStart; + file_offset_t samplestoreadinloop = Min(samplestoread, loopoffset); + file_offset_t reverseplaybackend = GetPos() - samplestoreadinloop; SetPos(reverseplaybackend); @@ -1014,11 +987,11 @@ // reading, swap all sample frames so it reflects // backward playback - unsigned long swapareastart = totalreadsamples; - unsigned long loopoffset = GetPos() - loop.LoopStart; - unsigned long samplestoreadinloop = (this->LoopPlayCount) ? Min(samplestoread, pPlaybackState->loop_cycles_left * loop.LoopLength - loopoffset) + file_offset_t swapareastart = totalreadsamples; + file_offset_t loopoffset = GetPos() - loop.LoopStart; + file_offset_t samplestoreadinloop = (this->LoopPlayCount) ? Min(samplestoread, pPlaybackState->loop_cycles_left * loop.LoopLength - loopoffset) : samplestoread; - unsigned long reverseplaybackend = loop.LoopStart + Abs((loopoffset - samplestoreadinloop) % loop.LoopLength); + file_offset_t reverseplaybackend = loop.LoopStart + Abs((loopoffset - samplestoreadinloop) % loop.LoopLength); SetPos(reverseplaybackend); @@ -1098,7 +1071,7 @@ * @returns number of successfully read sample points * @see SetPos(), CreateDecompressionBuffer() */ - unsigned long Sample::Read(void* pBuffer, unsigned long SampleCount, buffer_t* pExternalDecompressionBuffer) { + file_offset_t Sample::Read(void* pBuffer, file_offset_t SampleCount, buffer_t* pExternalDecompressionBuffer) { if (SampleCount == 0) return 0; if (!Compressed) { if (BitDepth == 24) { @@ -1113,7 +1086,7 @@ else { if (this->SamplePos >= this->SamplesTotal) return 0; //TODO: efficiency: maybe we should test for an average compression rate - unsigned long assumedsize = GuessSize(SampleCount), + file_offset_t assumedsize = GuessSize(SampleCount), remainingbytes = 0, // remaining bytes in the local buffer remainingsamples = SampleCount, copysamples, skipsamples, @@ -1136,8 +1109,8 @@ remainingbytes = pCkData->Read(pSrc, assumedsize, 1); while (remainingsamples && remainingbytes) { - unsigned long framesamples = SamplesPerFrame; - unsigned long framebytes, rightChannelOffset = 0, nextFrameOffset; + file_offset_t framesamples = SamplesPerFrame; + file_offset_t framebytes, rightChannelOffset = 0, nextFrameOffset; int mode_l = *pSrc++, mode_r = 0; @@ -1287,7 +1260,7 @@ * @throws gig::Exception if sample is compressed * @see DLS::LoadSampleData() */ - unsigned long Sample::Write(void* pBuffer, unsigned long SampleCount) { + file_offset_t Sample::Write(void* pBuffer, file_offset_t SampleCount) { if (Compressed) throw gig::Exception("There is no support for writing compressed gig samples (yet)"); // if this is the first write in this sample, reset the @@ -1296,7 +1269,7 @@ __resetCRC(crc); } if (GetSize() < SampleCount) throw Exception("Could not write sample data, current sample size to small"); - unsigned long res; + file_offset_t res; if (BitDepth == 24) { res = pCkData->Write(pBuffer, SampleCount * FrameSize, 1) / FrameSize; } else { // 16 bit @@ -1330,11 +1303,11 @@ * @returns allocated decompression buffer * @see DestroyDecompressionBuffer() */ - buffer_t Sample::CreateDecompressionBuffer(unsigned long MaxReadSize) { + buffer_t Sample::CreateDecompressionBuffer(file_offset_t MaxReadSize) { buffer_t result; const double worstCaseHeaderOverhead = (256.0 /*frame size*/ + 12.0 /*header*/ + 2.0 /*compression type flag (stereo)*/) / 256.0; - result.Size = (unsigned long) (double(MaxReadSize) * 3.0 /*(24 Bit)*/ * 2.0 /*stereo*/ * worstCaseHeaderOverhead); + result.Size = (file_offset_t) (double(MaxReadSize) * 3.0 /*(24 Bit)*/ * 2.0 /*stereo*/ * worstCaseHeaderOverhead); result.pStart = new int8_t[result.Size]; result.NullExtensionSize = 0; return result; @@ -1368,6 +1341,51 @@ return pGroup; } + /** + * Checks the integrity of this sample's raw audio wave data. Whenever a + * Sample's raw wave data is intentionally modified (i.e. by calling + * Write() and supplying the new raw audio wave form data) a CRC32 checksum + * is calculated and stored/updated for this sample, along to the sample's + * meta informations. + * + * Now by calling this method the current raw audio wave data is checked + * against the already stored CRC32 check sum in order to check whether the + * sample data had been damaged unintentionally for some reason. Since by + * calling this method always the entire raw audio wave data has to be + * read, verifying all samples this way may take a long time accordingly. + * And that's also the reason why the sample integrity is not checked by + * default whenever a gig file is loaded. So this method must be called + * explicitly to fulfill this task. + * + * @returns true if sample is OK or false if the sample is damaged + * @throws Exception if no checksum had been stored to disk for this + * sample yet, or on I/O issues + */ + bool Sample::VerifyWaveData() { + File* pFile = static_cast(GetParent()); + const uint32_t referenceCRC = pFile->GetSampleChecksum(this); + uint32_t crc = CalculateWaveDataChecksum(); + return crc == referenceCRC; + } + + uint32_t Sample::CalculateWaveDataChecksum() { + const size_t sz = 20*1024; // 20kB buffer size + std::vector buffer(sz); + buffer.resize(sz); + + const size_t n = sz / FrameSize; + SetPos(0); + uint32_t crc = 0; + __resetCRC(crc); + while (true) { + file_offset_t nRead = Read(&buffer[0], n); + if (nRead <= 0) break; + __calculateCRC(&buffer[0], nRead * FrameSize, crc); + } + __encodeCRC(crc); + return crc; + } + Sample::~Sample() { Instances--; if (!Instances && InternalDecompressionBuffer.Size) { @@ -1384,7 +1402,7 @@ // *************** DimensionRegion *************** // * - uint DimensionRegion::Instances = 0; + size_t DimensionRegion::Instances = 0; DimensionRegion::VelocityTableMap* DimensionRegion::pVelocityTables = NULL; DimensionRegion::DimensionRegion(Region* pParent, RIFF::List* _3ewl) : DLS::Sampler(_3ewl) { @@ -1752,10 +1770,12 @@ * * Usually there is absolutely no need to call this method explicitly. * It will be called automatically when File::Save() was called. + * + * @param pProgress - callback function for progress notification */ - void DimensionRegion::UpdateChunks() { + void DimensionRegion::UpdateChunks(progress_t* pProgress) { // first update base class's chunk - DLS::Sampler::UpdateChunks(); + DLS::Sampler::UpdateChunks(pProgress); RIFF::Chunk* wsmp = pParentList->GetSubChunk(CHUNK_ID_WSMP); uint8_t* pData = (uint8_t*) wsmp->LoadChunkData(); @@ -3026,9 +3046,10 @@ * Usually there is absolutely no need to call this method explicitly. * It will be called automatically when File::Save() was called. * + * @param pProgress - callback function for progress notification * @throws gig::Exception if samples cannot be dereferenced */ - void Region::UpdateChunks() { + void Region::UpdateChunks(progress_t* pProgress) { // in the gig format we don't care about the Region's sample reference // but we still have to provide some existing one to not corrupt the // file, so to avoid the latter we simply always assign the sample of @@ -3036,11 +3057,11 @@ pSample = pDimensionRegions[0]->pSample; // first update base class's chunks - DLS::Region::UpdateChunks(); + DLS::Region::UpdateChunks(pProgress); // update dimension region's chunks for (int i = 0; i < DimensionRegions; i++) { - pDimensionRegions[i]->UpdateChunks(); + pDimensionRegions[i]->UpdateChunks(pProgress); } File* pFile = (File*) GetParent()->GetParent(); @@ -3130,12 +3151,13 @@ int step = 1; for (int i = 0 ; i < veldim ; i++) step <<= pDimensionDefinitions[i].bits; int skipveldim = (step << pDimensionDefinitions[veldim].bits) - step; - int end = step * pDimensionDefinitions[veldim].zones; // loop through all dimension regions for all dimensions except the velocity dimension int dim[8] = { 0 }; for (int i = 0 ; i < DimensionRegions ; i++) { + const int end = i + step * pDimensionDefinitions[veldim].zones; + // create a velocity table for all cases where the velocity zone is zero if (pDimensionRegions[i]->DimensionUpperLimits[veldim] || pDimensionRegions[i]->VelocityUpperLimit) { // create the velocity table @@ -3166,6 +3188,7 @@ } } + // jump to the next case where the velocity zone is zero int j; int shift = 0; for (j = 0 ; j < Dimensions ; j++) { @@ -3652,6 +3675,33 @@ UpdateVelocityTable(); } + /** @brief Change type of an existing dimension. + * + * Alters the dimension type of a dimension already existing on this + * region. If there is currently no dimension on this Region with type + * @a oldType, then this call with throw an Exception. Likewise there are + * cases where the requested dimension type cannot be performed. For example + * if the new dimension type shall be gig::dimension_samplechannel, and the + * current dimension has more than 2 zones. In such cases an Exception is + * thrown as well. + * + * @param oldType - identifies the existing dimension to be changed + * @param newType - to which dimension type it should be changed to + * @throws gig::Exception if requested change cannot be performed + */ + void Region::SetDimensionType(dimension_t oldType, dimension_t newType) { + if (oldType == newType) return; + dimension_def_t* def = GetDimensionDefinition(oldType); + if (!def) + throw gig::Exception("No dimension with provided old dimension type exists on this region"); + if (newType == dimension_samplechannel && def->zones != 2) + throw gig::Exception("Cannot change to dimension type 'sample channel', because existing dimension does not have 2 zones"); + if (GetDimensionDefinition(newType)) + throw gig::Exception("There is already a dimension with requested new dimension type on this region"); + def->dimension = newType; + def->split_type = __resolveSplitType(newType); + } + DimensionRegion* Region::GetDimensionRegionByBit(const std::map& DimCase) { uint8_t bits[8] = {}; for (std::map::const_iterator it = DimCase.begin(); @@ -3847,13 +3897,28 @@ if ((int32_t)WavePoolTableIndex == -1) return NULL; File* file = (File*) GetParent()->GetParent(); if (!file->pWavePoolTable) return NULL; - unsigned long soughtoffset = file->pWavePoolTable[WavePoolTableIndex]; - unsigned long soughtfileno = file->pWavePoolTableHi[WavePoolTableIndex]; - Sample* sample = file->GetFirstSample(pProgress); - while (sample) { - if (sample->ulWavePoolOffset == soughtoffset && - sample->FileNo == soughtfileno) return static_cast(sample); - sample = file->GetNextSample(); + // for new files or files >= 2 GB use 64 bit wave pool offsets + if (file->pRIFF->IsNew() || (file->pRIFF->GetCurrentFileSize() >> 31)) { + // use 64 bit wave pool offsets (treating this as large file) + uint64_t soughtoffset = + uint64_t(file->pWavePoolTable[WavePoolTableIndex]) | + uint64_t(file->pWavePoolTableHi[WavePoolTableIndex]) << 32; + Sample* sample = file->GetFirstSample(pProgress); + while (sample) { + if (sample->ullWavePoolOffset == soughtoffset) + return static_cast(sample); + sample = file->GetNextSample(); + } + } else { + // use extension files and 32 bit wave pool offsets + file_offset_t soughtoffset = file->pWavePoolTable[WavePoolTableIndex]; + file_offset_t soughtfileno = file->pWavePoolTableHi[WavePoolTableIndex]; + Sample* sample = file->GetFirstSample(pProgress); + while (sample) { + if (sample->ullWavePoolOffset == soughtoffset && + sample->FileNo == soughtfileno) return static_cast(sample); + sample = file->GetNextSample(); + } } return NULL; } @@ -4134,7 +4199,16 @@ memcpy(&data[0], &text[0], text.size()); } - void Script::UpdateChunks() { + /** + * Apply this script to the respective RIFF chunks. You have to call + * File::Save() to make changes persistent. + * + * Usually there is absolutely no need to call this method explicitly. + * It will be called automatically when File::Save() was called. + * + * @param pProgress - callback function for progress notification + */ + void Script::UpdateChunks(progress_t* pProgress) { // recalculate CRC32 check sum __resetCRC(crc); __calculateCRC(&data[0], data.size(), crc); @@ -4173,7 +4247,7 @@ * @param pGroup - script's new group */ void Script::SetGroup(ScriptGroup* pGroup) { - if (this->pGroup = pGroup) return; + if (this->pGroup == pGroup) return; if (pChunk) pChunk->GetParent()->MoveSubChunk(pChunk, pGroup->pList); this->pGroup = pGroup; @@ -4224,7 +4298,16 @@ } } - void ScriptGroup::UpdateChunks() { + /** + * Apply this script group to the respective RIFF chunks. You have to call + * File::Save() to make changes persistent. + * + * Usually there is absolutely no need to call this method explicitly. + * It will be called automatically when File::Save() was called. + * + * @param pProgress - callback function for progress notification + */ + void ScriptGroup::UpdateChunks(progress_t* pProgress) { if (pScripts) { if (!pList) pList = pFile->pRIFF->GetSubList(LIST_TYPE_3LS)->AddSubList(LIST_TYPE_RTIS); @@ -4235,7 +4318,7 @@ for (std::list::iterator it = pScripts->begin(); it != pScripts->end(); ++it) { - (*it)->UpdateChunks(); + (*it)->UpdateChunks(pProgress); } } } @@ -4398,15 +4481,19 @@ if (lst3LS) { RIFF::Chunk* ckSCSL = lst3LS->GetSubChunk(CHUNK_ID_SCSL); if (ckSCSL) { - int slotCount = ckSCSL->ReadUint32(); - int slotSize = ckSCSL->ReadUint32(); - int unknownSpace = slotSize - 2*sizeof(uint32_t); // in case of future extensions - for (int i = 0; i < slotCount; ++i) { - _ScriptPooolEntry e; - e.fileOffset = ckSCSL->ReadUint32(); - e.bypass = ckSCSL->ReadUint32() & 1; - if (unknownSpace) ckSCSL->SetPos(unknownSpace, RIFF::stream_curpos); // in case of future extensions - scriptPoolFileOffsets.push_back(e); + int headerSize = ckSCSL->ReadUint32(); + int slotCount = ckSCSL->ReadUint32(); + if (slotCount) { + int slotSize = ckSCSL->ReadUint32(); + ckSCSL->SetPos(headerSize); // in case of future header extensions + int unknownSpace = slotSize - 2*sizeof(uint32_t); // in case of future slot extensions + for (int i = 0; i < slotCount; ++i) { + _ScriptPooolEntry e; + e.fileOffset = ckSCSL->ReadUint32(); + e.bypass = ckSCSL->ReadUint32() & 1; + if (unknownSpace) ckSCSL->SetPos(unknownSpace, RIFF::stream_curpos); // in case of future extensions + scriptPoolFileOffsets.push_back(e); + } } } } @@ -4441,18 +4528,19 @@ * Usually there is absolutely no need to call this method explicitly. * It will be called automatically when File::Save() was called. * + * @param pProgress - callback function for progress notification * @throws gig::Exception if samples cannot be dereferenced */ - void Instrument::UpdateChunks() { + void Instrument::UpdateChunks(progress_t* pProgress) { // first update base classes' chunks - DLS::Instrument::UpdateChunks(); + DLS::Instrument::UpdateChunks(pProgress); // update Regions' chunks { RegionList::iterator iter = pRegions->begin(); RegionList::iterator end = pRegions->end(); for (; iter != end; ++iter) - (*iter)->UpdateChunks(); + (*iter)->UpdateChunks(pProgress); } // make sure 'lart' RIFF list chunk exists @@ -4489,27 +4577,64 @@ } // own gig format extensions - if (pScriptRefs) { + if (ScriptSlotCount()) { + // make sure we have converted the original loaded script file + // offsets into valid Script object pointers + LoadScripts(); + RIFF::List* lst3LS = pCkInstrument->GetSubList(LIST_TYPE_3LS); if (!lst3LS) lst3LS = pCkInstrument->AddSubList(LIST_TYPE_3LS); - const int totalSize = pScriptRefs->size() * 2*sizeof(uint32_t); + const int slotCount = pScriptRefs->size(); + const int headerSize = 3 * sizeof(uint32_t); + const int slotSize = 2 * sizeof(uint32_t); + const int totalChunkSize = headerSize + slotCount * slotSize; RIFF::Chunk* ckSCSL = lst3LS->GetSubChunk(CHUNK_ID_SCSL); - if (!ckSCSL) ckSCSL = lst3LS->AddSubChunk(CHUNK_ID_SCSL, totalSize); - else ckSCSL->Resize(totalSize); + if (!ckSCSL) ckSCSL = lst3LS->AddSubChunk(CHUNK_ID_SCSL, totalChunkSize); + else ckSCSL->Resize(totalChunkSize); uint8_t* pData = (uint8_t*) ckSCSL->LoadChunkData(); - for (int i = 0, pos = 0; i < pScriptRefs->size(); ++i) { - int fileOffset = - (*pScriptRefs)[i].script->pChunk->GetFilePos() - - (*pScriptRefs)[i].script->pChunk->GetPos() - - CHUNK_HEADER_SIZE; - store32(&pData[pos], fileOffset); + int pos = 0; + store32(&pData[pos], headerSize); + pos += sizeof(uint32_t); + store32(&pData[pos], slotCount); + pos += sizeof(uint32_t); + store32(&pData[pos], slotSize); + pos += sizeof(uint32_t); + for (int i = 0; i < slotCount; ++i) { + // arbitrary value, the actual file offset will be updated in + // UpdateScriptFileOffsets() after the file has been resized + int bogusFileOffset = 0; + store32(&pData[pos], bogusFileOffset); pos += sizeof(uint32_t); store32(&pData[pos], (*pScriptRefs)[i].bypass ? 1 : 0); pos += sizeof(uint32_t); } + } else { + // no script slots, so get rid of any LS custom RIFF chunks (if any) + RIFF::List* lst3LS = pCkInstrument->GetSubList(LIST_TYPE_3LS); + if (lst3LS) pCkInstrument->DeleteSubChunk(lst3LS); } } + void Instrument::UpdateScriptFileOffsets() { + // own gig format extensions + if (pScriptRefs && pScriptRefs->size() > 0) { + RIFF::List* lst3LS = pCkInstrument->GetSubList(LIST_TYPE_3LS); + RIFF::Chunk* ckSCSL = lst3LS->GetSubChunk(CHUNK_ID_SCSL); + const int slotCount = pScriptRefs->size(); + const int headerSize = 3 * sizeof(uint32_t); + ckSCSL->SetPos(headerSize); + for (int i = 0; i < slotCount; ++i) { + uint32_t fileOffset = + (*pScriptRefs)[i].script->pChunk->GetFilePos() - + (*pScriptRefs)[i].script->pChunk->GetPos() - + CHUNK_HEADER_SIZE(ckSCSL->GetFile()->GetFileOffsetSize()); + ckSCSL->WriteUint32(&fileOffset); + // jump over flags entry (containing the bypass flag) + ckSCSL->SetPos(sizeof(uint32_t), RIFF::stream_curpos); + } + } + } + /** * Returns the appropriate Region for a triggered note. * @@ -4577,6 +4702,63 @@ } /** + * Move this instrument at the position before @arg dst. + * + * This method can be used to reorder the sequence of instruments in a + * .gig file. This might be helpful especially on large .gig files which + * contain a large number of instruments within the same .gig file. So + * grouping such instruments to similar ones, can help to keep track of them + * when working with such complex .gig files. + * + * When calling this method, this instrument will be removed from in its + * current position in the instruments list and moved to the requested + * target position provided by @param dst. You may also pass NULL as + * argument to this method, in that case this intrument will be moved to the + * very end of the .gig file's instrument list. + * + * You have to call Save() to make the order change persistent to the .gig + * file. + * + * Currently this method is limited to moving the instrument within the same + * .gig file. Trying to move it to another .gig file by calling this method + * will throw an exception. + * + * @param dst - destination instrument at which this instrument will be + * moved to, or pass NULL for moving to end of list + * @throw gig::Exception if this instrument and target instrument are not + * part of the same file + */ + void Instrument::MoveTo(Instrument* dst) { + if (dst && GetParent() != dst->GetParent()) + throw Exception( + "gig::Instrument::MoveTo() can only be used for moving within " + "the same gig file." + ); + + File* pFile = (File*) GetParent(); + + // move this instrument within the instrument list + { + File::InstrumentList& list = *pFile->pInstruments; + + File::InstrumentList::iterator itFrom = + std::find(list.begin(), list.end(), static_cast(this)); + + File::InstrumentList::iterator itTo = + std::find(list.begin(), list.end(), static_cast(dst)); + + list.splice(itTo, list, itFrom); + } + + // move the instrument's actual list RIFF chunk appropriately + RIFF::List* lstCkInstruments = pFile->pRIFF->GetSubList(LIST_TYPE_LINS); + lstCkInstruments->MoveSubChunk( + this->pCkInstrument, + (RIFF::Chunk*) ((dst) ? dst->pCkInstrument : NULL) + ); + } + + /** * Returns a MIDI rule of the instrument. * * The list of MIDI rules, at least in gig v3, always contains at @@ -4645,16 +4827,16 @@ if (scriptPoolFileOffsets.empty()) return; File* pFile = (File*) GetParent(); for (uint k = 0; k < scriptPoolFileOffsets.size(); ++k) { - uint32_t offset = scriptPoolFileOffsets[k].fileOffset; + uint32_t soughtOffset = scriptPoolFileOffsets[k].fileOffset; for (uint i = 0; pFile->GetScriptGroup(i); ++i) { ScriptGroup* group = pFile->GetScriptGroup(i); for (uint s = 0; group->GetScript(s); ++s) { Script* script = group->GetScript(s); if (script->pChunk) { - script->pChunk->SetPos(0); - if (script->pChunk->GetFilePos() - - script->pChunk->GetPos() - - CHUNK_HEADER_SIZE == offset) + uint32_t offset = script->pChunk->GetFilePos() - + script->pChunk->GetPos() - + CHUNK_HEADER_SIZE(script->pChunk->GetFile()->GetFileOffsetSize()); + if (offset == soughtOffset) { _ScriptPooolRef ref; ref.script = script; @@ -4936,8 +5118,10 @@ * * Usually there is absolutely no need to call this method explicitly. * It will be called automatically when File::Save() was called. + * + * @param pProgress - callback function for progress notification */ - void Group::UpdateChunks() { + void Group::UpdateChunks(progress_t* pProgress) { // make sure <3gri> and <3gnl> list chunks exist RIFF::List* _3gri = pFile->pRIFF->GetSubList(LIST_TYPE_3GRI); if (!_3gri) { @@ -5214,9 +5398,12 @@ int iTotalSamples = WavePoolCount; // check if samples should be loaded from extension files + // (only for old gig files < 2 GB) int lastFileNo = 0; - for (int i = 0 ; i < WavePoolCount ; i++) { - if (pWavePoolTableHi[i] > lastFileNo) lastFileNo = pWavePoolTableHi[i]; + if (!file->IsNew() && !(file->GetCurrentFileSize() >> 31)) { + for (int i = 0 ; i < WavePoolCount ; i++) { + if (pWavePoolTableHi[i] > lastFileNo) lastFileNo = pWavePoolTableHi[i]; + } } String name(pRIFF->GetFileName()); int nameLen = name.length(); @@ -5226,7 +5413,7 @@ for (int fileNo = 0 ; ; ) { RIFF::List* wvpl = file->GetSubList(LIST_TYPE_WVPL); if (wvpl) { - unsigned long wvplFileOffset = wvpl->GetFilePos(); + file_offset_t wvplFileOffset = wvpl->GetFilePos(); RIFF::List* wave = wvpl->GetFirstSubList(); while (wave) { if (wave->GetListType() == LIST_TYPE_WAVE) { @@ -5234,7 +5421,7 @@ const float subprogress = (float) iSampleIndex / (float) iTotalSamples; __notify_progress(pProgress, subprogress); - unsigned long waveFileOffset = wave->GetFilePos(); + file_offset_t waveFileOffset = wave->GetFilePos(); pSamples->push_back(new Sample(this, wave, waveFileOffset - wvplFileOffset, fileNo)); iSampleIndex++; @@ -5473,22 +5660,141 @@ if (!_3crc) return; // get the index of the sample - int iWaveIndex = -1; - File::SampleList::iterator iter = pSamples->begin(); - File::SampleList::iterator end = pSamples->end(); - for (int index = 0; iter != end; ++iter, ++index) { - if (*iter == pSample) { - iWaveIndex = index; - break; - } - } + int iWaveIndex = GetWaveTableIndexOf(pSample); if (iWaveIndex < 0) throw gig::Exception("Could not update crc, could not find sample"); // write the CRC-32 checksum to disk _3crc->SetPos(iWaveIndex * 8); - uint32_t tmp = 1; - _3crc->WriteUint32(&tmp); // unknown, always 1? + uint32_t one = 1; + _3crc->WriteUint32(&one); // always 1 _3crc->WriteUint32(&crc); + + // reload CRC table to RAM to keep it persistent over several subsequent save operations + _3crc->ReleaseChunkData(); + _3crc->LoadChunkData(); + } + + uint32_t File::GetSampleChecksum(Sample* pSample) { + RIFF::Chunk* _3crc = pRIFF->GetSubChunk(CHUNK_ID_3CRC); + if (!_3crc) throw gig::Exception("Could not retrieve reference crc of sample, no checksums stored for this file yet"); + uint8_t* pData = (uint8_t*) _3crc->LoadChunkData(); + if (!pData) throw gig::Exception("Could not retrieve reference crc of sample, no checksums stored for this file yet"); + + // get the index of the sample + int iWaveIndex = GetWaveTableIndexOf(pSample); + if (iWaveIndex < 0) throw gig::Exception("Could not retrieve reference crc of sample, could not resolve sample's wave table index"); + + // read the CRC-32 checksum directly from disk + size_t pos = iWaveIndex * 8; + if (pos + 8 > _3crc->GetNewSize()) + throw gig::Exception("Could not retrieve reference crc of sample, could not seek to required position in crc chunk"); + + uint32_t one = load32(&pData[pos]); // always 1 + if (one != 1) + throw gig::Exception("Could not verify sample, because reference checksum table is damaged"); + + return load32(&pData[pos+4]); + } + + int File::GetWaveTableIndexOf(gig::Sample* pSample) { + if (!pSamples) GetFirstSample(); // make sure sample chunks were scanned + File::SampleList::iterator iter = pSamples->begin(); + File::SampleList::iterator end = pSamples->end(); + for (int index = 0; iter != end; ++iter, ++index) + if (*iter == pSample) + return index; + return -1; + } + + /** + * Checks whether the file's "3CRC" chunk was damaged. This chunk contains + * the CRC32 check sums of all samples' raw wave data. + * + * @return true if 3CRC chunk is OK, or false if 3CRC chunk is damaged + */ + bool File::VerifySampleChecksumTable() { + RIFF::Chunk* _3crc = pRIFF->GetSubChunk(CHUNK_ID_3CRC); + if (!_3crc) return false; + if (_3crc->GetNewSize() <= 0) return false; + if (_3crc->GetNewSize() % 8) return false; + if (!pSamples) GetFirstSample(); // make sure sample chunks were scanned + if (_3crc->GetNewSize() != pSamples->size() * 8) return false; + + const int n = _3crc->GetNewSize() / 8; + + uint32_t* pData = (uint32_t*) _3crc->LoadChunkData(); + if (!pData) return false; + + for (int i = 0; i < n; ++i) { + uint32_t one = pData[i*2]; + if (one != 1) return false; + } + + return true; + } + + /** + * Recalculates CRC32 checksums for all samples and rebuilds this gig + * file's checksum table with those new checksums. This might usually + * just be necessary if the checksum table was damaged. + * + * @e IMPORTANT: The current implementation of this method only works + * with files that have not been modified since it was loaded, because + * it expects that no externally caused file structure changes are + * required! + * + * Due to the expectation above, this method is currently protected + * and actually only used by the command line tool "gigdump" yet. + * + * @returns true if Save() is required to be called after this call, + * false if no further action is required + */ + bool File::RebuildSampleChecksumTable() { + // make sure sample chunks were scanned + if (!pSamples) GetFirstSample(); + + bool bRequiresSave = false; + + // make sure "3CRC" chunk exists with required size + RIFF::Chunk* _3crc = pRIFF->GetSubChunk(CHUNK_ID_3CRC); + if (!_3crc) { + _3crc = pRIFF->AddSubChunk(CHUNK_ID_3CRC, pSamples->size() * 8); + bRequiresSave = true; + } else if (_3crc->GetNewSize() != pSamples->size() * 8) { + _3crc->Resize(pSamples->size() * 8); + bRequiresSave = true; + } + + if (bRequiresSave) { // refill CRC table for all samples in RAM ... + uint32_t* pData = (uint32_t*) _3crc->LoadChunkData(); + { + File::SampleList::iterator iter = pSamples->begin(); + File::SampleList::iterator end = pSamples->end(); + for (; iter != end; ++iter) { + gig::Sample* pSample = (gig::Sample*) *iter; + int index = GetWaveTableIndexOf(pSample); + if (index < 0) throw gig::Exception("Could not rebuild crc table for samples, wave table index of a sample could not be resolved"); + pData[index*2] = 1; // always 1 + pData[index*2+1] = pSample->CalculateWaveDataChecksum(); + } + } + } else { // no file structure changes necessary, so directly write to disk and we are done ... + // make sure file is in write mode + pRIFF->SetMode(RIFF::stream_mode_read_write); + { + File::SampleList::iterator iter = pSamples->begin(); + File::SampleList::iterator end = pSamples->end(); + for (; iter != end; ++iter) { + gig::Sample* pSample = (gig::Sample*) *iter; + int index = GetWaveTableIndexOf(pSample); + if (index < 0) throw gig::Exception("Could not rebuild crc table for samples, wave table index of a sample could not be resolved"); + uint32_t crc = pSample->CalculateWaveDataChecksum(); + SetSampleChecksum(pSample, crc); + } + } + } + + return bRequiresSave; } Group* File::GetFirstGroup() { @@ -5713,38 +6019,50 @@ * Usually there is absolutely no need to call this method explicitly. * It will be called automatically when File::Save() was called. * + * @param pProgress - callback function for progress notification * @throws Exception - on errors */ - void File::UpdateChunks() { + void File::UpdateChunks(progress_t* pProgress) { bool newFile = pRIFF->GetSubList(LIST_TYPE_INFO) == NULL; - b64BitWavePoolOffsets = pVersion && pVersion->major == 3; - // update own gig format extension chunks // (not part of the GigaStudio 4 format) - // + RIFF::List* lst3LS = pRIFF->GetSubList(LIST_TYPE_3LS); + if (!lst3LS) { + lst3LS = pRIFF->AddSubList(LIST_TYPE_3LS); + } + // Make sure <3LS > chunk is placed before chunk. The precise + // location of <3LS > is irrelevant, however it should be located + // before the actual wave data + RIFF::Chunk* ckPTBL = pRIFF->GetSubChunk(CHUNK_ID_PTBL); + pRIFF->MoveSubChunk(lst3LS, ckPTBL); + // This must be performed before writing the chunks for instruments, // because the instruments' script slots will write the file offsets // of the respective instrument script chunk as reference. if (pScriptGroups) { - RIFF::List* lst3LS = pRIFF->GetSubList(LIST_TYPE_3LS); - if (pScriptGroups->empty()) { - if (lst3LS) pRIFF->DeleteSubChunk(lst3LS); - } else { - if (!lst3LS) lst3LS = pRIFF->AddSubList(LIST_TYPE_3LS); - - // Update instrument script (group) chunks. - - for (std::list::iterator it = pScriptGroups->begin(); - it != pScriptGroups->end(); ++it) - { - (*it)->UpdateChunks(); - } + // Update instrument script (group) chunks. + for (std::list::iterator it = pScriptGroups->begin(); + it != pScriptGroups->end(); ++it) + { + (*it)->UpdateChunks(pProgress); } } + // in case no libgig custom format data was added, then remove the + // custom "3LS " chunk again + if (!lst3LS->CountSubChunks()) { + pRIFF->DeleteSubChunk(lst3LS); + lst3LS = NULL; + } + + // if there is a 3CRC chunk, make sure it is loaded into RAM, otherwise + // its data might get lost or damaged on file structure changes + RIFF::Chunk* _3crc = pRIFF->GetSubChunk(CHUNK_ID_3CRC); + if (_3crc) _3crc->LoadChunkData(); + // first update base class's chunks - DLS::File::UpdateChunks(); + DLS::File::UpdateChunks(pProgress); if (newFile) { // INFO was added by Resource::UpdateChunks - make sure it @@ -5781,7 +6099,7 @@ std::list::iterator iter = pGroups->begin(); std::list::iterator end = pGroups->end(); for (; iter != end; ++iter) { - (*iter)->UpdateChunks(); + (*iter)->UpdateChunks(pProgress); } } @@ -5902,7 +6220,6 @@ // samples. The actual checksum values will be filled in // later, by Sample::Write. - RIFF::Chunk* _3crc = pRIFF->GetSubChunk(CHUNK_ID_3CRC); if (_3crc) { _3crc->Resize(pSamples->size() * 8); } else if (newFile) { @@ -5913,6 +6230,16 @@ if (einf && pVersion && pVersion->major == 3) pRIFF->MoveSubChunk(_3crc, einf); } } + + void File::UpdateFileOffsets() { + DLS::File::UpdateFileOffsets(); + + for (Instrument* instrument = GetFirstInstrument(); instrument; + instrument = GetNextInstrument()) + { + instrument->UpdateScriptFileOffsets(); + } + } /** * Enable / disable automatic loading. By default this properyt is