77 if ( AT.SS != AT.S0 )
return(0);
78 if ( AR.NoCompress == 1 )
return(0);
79 if ( AR.gzipCompress <= 0 )
return(0);
81 if ( f->ziobuffer == 0 ) {
85 f->zsp = Malloc1(
sizeof(z_stream),
"output zstream");
90 (Bytef *)Malloc1(f->ziosize*
sizeof(
char),
"output zbuffer");
91 if ( f->zsp == 0 || f->ziobuffer == 0 ) {
92 MLOCK(ErrorMessageLock);
93 MesCall(
"SetupOutputGZIP");
94 MUNLOCK(ErrorMessageLock);
101 f->zsp->zalloc = Z_NULL;
102 f->zsp->zfree = Z_NULL;
103 f->zsp->opaque = Z_NULL;
107 f->zsp->next_out = f->ziobuffer;
108 f->zsp->avail_out = f->ziosize;
109 f->zsp->total_out = 0;
113 f->zsp->next_in = (Bytef *)(f->PObuffer);
114 f->zsp->avail_in = (Bytef *)(f->POfill) - (Bytef *)(f->PObuffer);
115 f->zsp->total_in = 0;
119 if ( deflateInit(f->zsp,AR.gzipCompress) != Z_OK ) {
120 MLOCK(ErrorMessageLock);
121 MesPrint(
"Error from zlib: %s",f->zsp->msg);
122 MesCall(
"SetupOutputGZIP");
123 MUNLOCK(ErrorMessageLock);
148 f->zsp->next_in = (Bytef *)(f->PObuffer);
149 f->zsp->avail_in = (Bytef *)(f->POfill) - (Bytef *)(f->PObuffer);
150 f->zsp->total_in = 0;
152 while ( ( zerror = deflate(f->zsp,Z_NO_FLUSH) ) == Z_OK ) {
153 if ( f->zsp->avail_out == 0 ) {
159 char *s = (
char *)((UBYTE *)(f->ziobuffer)+f->ziosize);
160 MLOCK(ErrorMessageLock);
161 MesPrint(
"%wWriting %l bytes at %10p: %d %d %d %d %d" 162 ,f->ziosize,&(f->POposition),s[-5],s[-4],s[-3],s[-2],s[-1]);
163 MUNLOCK(ErrorMessageLock);
167 LOCK(f->pthreadslock);
169 if ( f == AR.hidefile ) {
172 SeekFile(f->
handle,&(f->POposition),SEEK_SET);
173 if ( WriteFile(f->
handle,(UBYTE *)(f->ziobuffer),f->ziosize)
175 if ( f == AR.hidefile ) {
176 UNLOCK(AS.inputslock);
179 UNLOCK(f->pthreadslock);
181 MLOCK(ErrorMessageLock);
182 MesPrint(
"%wWrite error during compressed sort. Disk full?");
183 MUNLOCK(ErrorMessageLock);
186 if ( f == AR.hidefile ) {
187 UNLOCK(AS.inputslock);
190 UNLOCK(f->pthreadslock);
192 ADDPOS(f->filesize,f->ziosize);
193 ADDPOS(f->POposition,f->ziosize);
195 if ( AS.MasterSort && AC.ThreadSortFileSynch ) {
202 f->zsp->next_out = f->ziobuffer;
203 f->zsp->avail_out = f->ziosize;
204 f->zsp->total_out = 0;
206 else if ( f->zsp->avail_in == 0 ) {
213 MLOCK(ErrorMessageLock);
214 MesPrint(
"%w avail_in = %d, avail_out = %d.",f->zsp->avail_in,f->zsp->avail_out);
215 MUNLOCK(ErrorMessageLock);
219 MLOCK(ErrorMessageLock);
220 MesPrint(
"%wError in gzip handling of output. zerror = %d",zerror);
221 MUNLOCK(ErrorMessageLock);
241 f->zsp->next_in = (Bytef *)(f->PObuffer);
242 f->zsp->avail_in = (Bytef *)(f->POfill) - (Bytef *)(f->PObuffer);
243 f->zsp->total_in = 0;
245 while ( ( zerror = deflate(f->zsp,Z_FINISH) ) == Z_OK ) {
246 if ( f->zsp->avail_out == 0 ) {
251 MLOCK(ErrorMessageLock);
252 MesPrint(
"%wWriting %l bytes at %10p",f->ziosize,&(f->POposition));
253 MUNLOCK(ErrorMessageLock);
256 LOCK(f->pthreadslock);
258 if ( f == AR.hidefile ) {
259 UNLOCK(AS.inputslock);
261 SeekFile(f->
handle,&(f->POposition),SEEK_SET);
262 if ( WriteFile(f->
handle,(UBYTE *)(f->ziobuffer),f->ziosize)
264 if ( f == AR.hidefile ) {
265 UNLOCK(AS.inputslock);
268 UNLOCK(f->pthreadslock);
270 MLOCK(ErrorMessageLock);
271 MesPrint(
"%wWrite error during compressed sort. Disk full?");
272 MUNLOCK(ErrorMessageLock);
275 if ( f == AR.hidefile ) {
276 UNLOCK(AS.inputslock);
279 UNLOCK(f->pthreadslock);
281 ADDPOS(f->filesize,f->ziosize);
282 ADDPOS(f->POposition,f->ziosize);
284 if ( AS.MasterSort && AC.ThreadSortFileSynch ) {
291 f->zsp->next_out = f->ziobuffer;
292 f->zsp->avail_out = f->ziosize;
293 f->zsp->total_out = 0;
296 if ( zerror == Z_STREAM_END ) {
301 MLOCK(ErrorMessageLock);
302 MesPrint(
"%wWriting %l bytes at %10p",(LONG)(f->zsp->avail_out),&(f->POposition));
303 MUNLOCK(ErrorMessageLock);
306 LOCK(f->pthreadslock);
308 if ( f == AR.hidefile ) {
311 SeekFile(f->
handle,&(f->POposition),SEEK_SET);
312 if ( WriteFile(f->
handle,(UBYTE *)(f->ziobuffer),f->zsp->total_out)
313 != (LONG)(f->zsp->total_out) ) {
314 if ( f == AR.hidefile ) {
315 UNLOCK(AS.inputslock);
318 UNLOCK(f->pthreadslock);
320 MLOCK(ErrorMessageLock);
321 MesPrint(
"%wWrite error during compressed sort. Disk full?");
322 MUNLOCK(ErrorMessageLock);
325 if ( f == AR.hidefile ) {
329 UNLOCK(f->pthreadslock);
331 ADDPOS(f->filesize,f->zsp->total_out);
332 ADDPOS(f->POposition,f->zsp->total_out);
334 if ( AS.MasterSort && AC.ThreadSortFileSynch ) {
339 MLOCK(ErrorMessageLock);
340 {
char *s = f->ziobuffer+f->zsp->total_out;
341 MesPrint(
"%w Last bytes written: %d %d %d %d %d",s[-5],s[-4],s[-3],s[-2],s[-1]);
343 MesPrint(
"%w Perceived position in FlushOutputGZIP is %10p",&(f->POposition));
344 MUNLOCK(ErrorMessageLock);
349 f->zsp->next_out = f->ziobuffer;
350 f->zsp->avail_out = f->ziosize;
351 f->zsp->total_out = 0;
352 if ( ( zerror = deflateEnd(f->zsp) ) == Z_OK )
return(0);
353 MLOCK(ErrorMessageLock);
355 MesPrint(
"%wError in finishing gzip handling of output: %s",f->zsp->msg);
358 MesPrint(
"%wError in finishing gzip handling of output.");
360 MUNLOCK(ErrorMessageLock);
363 MLOCK(ErrorMessageLock);
364 MesPrint(
"%wError in gzip handling of output.");
365 MUNLOCK(ErrorMessageLock);
381 int SetupAllInputGZIP(
SORTING *S)
384 int i, NumberOpened = 0;
390 for ( i = 0; i < S->inNum; i++ ) {
391 if ( S->fpincompressed[i] )
break;
393 if ( i >= S->inNum )
return(0);
395 if ( S->zsparray == 0 ) {
396 S->zsparray = (z_streamp)Malloc1(
sizeof(z_stream)*S->MaxFpatches,
"input zstreams");
397 if ( S->zsparray == 0 ) {
398 MLOCK(ErrorMessageLock);
399 MesCall(
"SetupAllInputGZIP");
400 MUNLOCK(ErrorMessageLock);
407 AN.ziobuffers = (Bytef *)Malloc1(S->MaxFpatches*(S->file.ziosize+128)*
sizeof(Bytef),
"input raw buffers");
413 AN.ziobufnum = (Bytef **)Malloc1(S->MaxFpatches*
sizeof(Bytef *),
"input raw pointers");
414 if ( AN.ziobuffers == 0 || AN.ziobufnum == 0 ) {
415 MLOCK(ErrorMessageLock);
416 MesCall(
"SetupAllInputGZIP");
417 MUNLOCK(ErrorMessageLock);
420 for ( i = 0 ; i < S->MaxFpatches; i++ ) {
421 AN.ziobufnum[i] = AN.ziobuffers + i * (S->file.ziosize+128);
424 for ( i = 0; i < S->inNum; i++ ) {
426 MLOCK(ErrorMessageLock);
427 MesPrint(
"%wPreparing z-stream %d with compression %d",i,S->fpincompressed[i]);
428 MUNLOCK(ErrorMessageLock);
430 if ( S->fpincompressed[i] ) {
431 zsp = &(S->zsparray[i]);
435 zsp->zalloc = Z_NULL;
437 zsp->opaque = Z_NULL;
441 zsp->next_out = Z_NULL;
447 zsp->next_in = Z_NULL;
453 if ( inflateInit(zsp) != Z_OK ) {
454 MLOCK(ErrorMessageLock);
455 if ( zsp->msg ) MesPrint(
"%wError from inflateInit: %s",zsp->msg);
456 else MesPrint(
"%wError from inflateInit");
457 MesCall(
"SetupAllInputGZIP");
458 MUNLOCK(ErrorMessageLock);
464 return(NumberOpened);
477 LONG FillInputGZIP(
FILEHANDLE *f,
POSITION *position, UBYTE *buffer, LONG buffersize,
int numstream)
481 LONG readsize, toread = 0;
485 if ( S->fpincompressed[numstream] ) {
486 zsp = &(S->zsparray[numstream]);
487 zsp->next_out = (Bytef *)buffer;
488 zsp->avail_out = buffersize;
490 if ( zsp->avail_in == 0 ) {
494 if ( ISGEPOSINC(S->fPatchesStop[numstream],*position,f->ziosize) ) {
498 DIFPOS(pos,S->fPatchesStop[numstream],*position);
499 toread = (LONG)(BASEPOSITION(pos));
503 MLOCK(ErrorMessageLock);
504 MesPrint(
"%w-+Reading %l bytes in stream %d at position %10p; stop at %10p",toread,numstream,position,&(S->fPatchesStop[numstream]));
505 MUNLOCK(ErrorMessageLock);
508 LOCK(f->pthreadslock);
510 SeekFile(f->
handle,position,SEEK_SET);
511 readsize = ReadFile(f->
handle,(UBYTE *)(AN.ziobufnum[numstream]),toread);
512 SeekFile(f->
handle,position,SEEK_CUR);
514 UNLOCK(f->pthreadslock);
517 MLOCK(ErrorMessageLock);
518 {
char *s = AN.ziobufnum[numstream]+readsize;
519 MesPrint(
"%w read: %l +Last bytes read: %d %d %d %d %d in %s, newpos = %10p",readsize,s[-5],s[-4],s[-3],s[-2],s[-1],f->name,position);
521 MUNLOCK(ErrorMessageLock);
523 if ( readsize == 0 ) {
524 zsp->next_in = AN.ziobufnum[numstream];
525 zsp->avail_in = f->ziosize;
527 return(zsp->total_out);
529 if ( readsize < 0 ) {
530 MLOCK(ErrorMessageLock);
531 MesPrint(
"%wFillInputGZIP: Read error during compressed sort.");
532 MUNLOCK(ErrorMessageLock);
535 ADDPOS(f->filesize,readsize);
536 ADDPOS(f->POposition,readsize);
540 zsp->next_in = AN.ziobufnum[numstream];
541 zsp->avail_in = readsize;
545 if ( toread > 0 || zsp->avail_in ) {
546 while ( ( zerror = inflate(zsp,Z_NO_FLUSH) ) == Z_OK ) {
547 if ( zsp->avail_out == 0 ) {
551 return((LONG)(zsp->total_out));
553 if ( zsp->avail_in == 0 ) {
555 if ( ISEQUALPOS(S->fPatchesStop[numstream],*position) ) {
559 zerror = Z_STREAM_END;
592 if ( numstream == 0 ) {
593 MLOCK(ErrorMessageLock);
594 MesPrint(
"%wWant to read in stream 0 at position %10p",position);
595 MUNLOCK(ErrorMessageLock);
598 if ( ISGEPOSINC(S->fPatchesStop[numstream],*position,f->ziosize) ) {
602 DIFPOS(pos,S->fPatchesStop[numstream],*position);
603 toread = (LONG)(BASEPOSITION(pos));
606 MLOCK(ErrorMessageLock);
607 MesPrint(
"%w--Reading %l bytes in stream %d at position %10p",toread,numstream,position);
608 MUNLOCK(ErrorMessageLock);
611 LOCK(f->pthreadslock);
613 SeekFile(f->
handle,position,SEEK_SET);
614 readsize = ReadFile(f->
handle,(UBYTE *)(AN.ziobufnum[numstream]),toread);
615 SeekFile(f->
handle,position,SEEK_CUR);
617 UNLOCK(f->pthreadslock);
620 MLOCK(ErrorMessageLock);
621 {
char *s = AN.ziobufnum[numstream]+readsize;
622 MesPrint(
"%w Last bytes read: %d %d %d %d %d",s[-5],s[-4],s[-3],s[-2],s[-1]);
624 MUNLOCK(ErrorMessageLock);
626 if ( readsize == 0 ) {
627 zsp->next_in = AN.ziobufnum[numstream];
628 zsp->avail_in = f->ziosize;
630 return(zsp->total_out);
632 if ( readsize < 0 ) {
633 MLOCK(ErrorMessageLock);
634 MesPrint(
"%wFillInputGZIP: Read error during compressed sort.");
635 MUNLOCK(ErrorMessageLock);
638 ADDPOS(f->filesize,readsize);
639 ADDPOS(f->POposition,readsize);
643 zsp->next_in = AN.ziobufnum[numstream];
644 zsp->avail_in = readsize;
653 zerror = Z_STREAM_END;
657 MLOCK(ErrorMessageLock);
658 MesPrint(
"%w zerror = %d in stream %d. At position %10p",zerror,numstream,position);
659 MUNLOCK(ErrorMessageLock);
661 if ( zerror == Z_STREAM_END ) {
665 zsp->next_in = Z_NULL;
672 MLOCK(ErrorMessageLock);
673 MesPrint(
"%wClosing stream %d",numstream);
675 readsize = zsp->total_out;
677 if ( readsize > 0 ) {
678 WORD *s = (WORD *)(buffer+zsp->total_out);
679 MesPrint(
"%w -Last words: %d %d %d %d %d",s[-5],s[-4],s[-3],s[-2],s[-1]);
682 MesPrint(
"%w No words");
684 MUNLOCK(ErrorMessageLock);
686 if ( zsp->zalloc != Z_NULL ) {
687 zerror = inflateEnd(zsp);
688 zsp->zalloc = Z_NULL;
690 if ( zerror == Z_OK || zerror == Z_STREAM_END )
return(readsize);
693 MLOCK(ErrorMessageLock);
694 MesPrint(
"%wFillInputGZIP: Error in gzip handling of input. zerror = %d",zerror);
695 MUNLOCK(ErrorMessageLock);
700 MLOCK(ErrorMessageLock);
701 MesPrint(
"%w++Reading %l bytes at position %10p",buffersize,position);
702 MUNLOCK(ErrorMessageLock);
705 LOCK(f->pthreadslock);
707 SeekFile(f->
handle,position,SEEK_SET);
708 readsize = ReadFile(f->
handle,buffer,buffersize);
709 SeekFile(f->
handle,position,SEEK_CUR);
711 UNLOCK(f->pthreadslock);
713 if ( readsize < 0 ) {
714 MLOCK(ErrorMessageLock);
715 MesPrint(
"%wFillInputGZIP: Read error during uncompressed sort.");
716 MesPrint(
"%w++Reading %l bytes at position %10p",buffersize,position);
717 MUNLOCK(ErrorMessageLock);
730 if ( f->ziobuffer ) {
731 M_free(f->ziobuffer,
"output zbuffer");
732 M_free(f->zsp,
"output zstream");