5  * This source code is licensed under both the BSD-style license (found in the
8  * You may select, at your option, one of the above-listed licenses.
21 /*-*************************************
37 /*-*************************************
48 static const size_t g_maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << siz…
54 /*-*************************************
68 /*-*************************************
93   If error returns -1.
98     return (fileSize == UTIL_FILESIZE_UNKNOWN) ? -1 : (S64)fileSize;  in DiB_getFileSize()
105  *  load samples from files listed in fileNamesTable into buffer.
106  *  works even if buffer is too small to load all samples.
109  * @return : nb of samples effectively loaded into `buffer`
130         if (fileSize <= 0) /* skip if zero-size or file error */  in DiB_loadFiles()
149         /* If file-chunking is enabled, load the rest of the file as more samples */  in DiB_loadFiles()
152                 size_t const chunkSize = MIN((size_t)(fileSize-fileDataLoaded), targetChunkSize);  in DiB_loadFiles()
170     DISPLAYLEVEL(4, "Loaded %d KB total training data, %d nb samples \n",  in DiB_loadFiles()
176 #define DiB_rotl32(x,r) ((x << r) | (x >> (32 - r)))
190  * shuffle a table of file names in a semi-random way
197     for (i = nbFiles - 1; i > 0; --i) {  in DiB_shuffle()
206 /*-********************************************************
220         requiredMem -= step;  in DiB_findMaxMem()
263  *  provides the amount of data to be loaded and the resulting nb of samples.
277       // TODO: is there a minimum sample size? What if the file is 1-byte?  in DiB_fileStats()
283       /* the case where we are breaking up files in sample chunks */  in DiB_fileStats()
286         // TODO: is there a minimum sample size? Can we have a 1-byte sample?  in DiB_fileStats()
287         fs.nbSamples += (int)((fileSize + chunkSize-1) / chunkSize);  in DiB_fileStats()
304 …DISPLAYLEVEL(4, "Found training data %d files, %d KB, %d samples\n", nbFiles, (int)(fs.totalSizeTo…  in DiB_fileStats()
314     size_t* sampleSizes; /* vector of sample sizes. Each sample can be up to SAMPLESIZE_MAX */  in DiB_trainFromFiles()
315     int nbSamplesLoaded; /* nb of samples effectively loaded in srcBuffer */  in DiB_trainFromFiles()
316     size_t loadedSize; /* total data loaded in srcBuffer for all samples */  in DiB_trainFromFiles()
317     void* srcBuffer /* contiguous buffer with training data/samples */;  in DiB_trainFromFiles()
321     int const displayLevel = params ? params->zParams.notificationLevel :  in DiB_trainFromFiles()
322         coverParams ? coverParams->zParams.notificationLevel :  in DiB_trainFromFiles()
323         fastCoverParams ? fastCoverParams->zParams.notificationLevel : 0;  in DiB_trainFromFiles()
326        The purpose of the shuffle is to pick random samples when the sample  in DiB_trainFromFiles()
331     /* Figure out how much sample data to load with how many samples */  in DiB_trainFromFiles()
357         DISPLAYLEVEL(2, "!  Note that dictionary is only useful for small samples. \n");  in DiB_trainFromFiles()
361         DISPLAYLEVEL(2, "!  Warning : nb of samples too low for proper processing ! \n");  in DiB_trainFromFiles()
363 …LAYLEVEL(2, "!  Alternatively, split files into fixed-size blocks representative of samples, with …  in DiB_trainFromFiles()
364         EXM_THROW(14, "nb of samples too low");   /* we now clearly forbid this case */  in DiB_trainFromFiles()
367 …      DISPLAYLEVEL(2, "!  Warning : data size of samples too small for target dictionary size \n");  in DiB_trainFromFiles()
368         DISPLAYLEVEL(2, "!  Samples should be about 100x larger than target dictionary size \n");  in DiB_trainFromFiles()
373         DISPLAYLEVEL(1, "Training samples set too large (%u MB); training on %u MB only...\n",  in DiB_trainFromFiles()
394                   unsigned splitPercentage = (unsigned)(coverParams->splitPoint * 100);  in DiB_trainFromFiles()
395 …                DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParams->k, coverParams->d,  in DiB_trainFromFiles()
396                               coverParams->steps, splitPercentage);  in DiB_trainFromFiles()
409                 unsigned splitPercentage = (unsigned)(fastCoverParams->splitPoint * 100);  in DiB_trainFromFiles()
410 …            DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", fastCoverParams->k,  in DiB_trainFromFiles()
411 …                   fastCoverParams->d, fastCoverParams->f, fastCoverParams->steps, splitPercentage,  in DiB_trainFromFiles()
412                             fastCoverParams->accel);  in DiB_trainFromFiles()
429     /* clean up */  in DiB_trainFromFiles()