text: /*-------------------------------------------------------------------------
   *
   * pg_backup_tar.c
   *
   *	This file is copied from the 'files' format file, but dumps data into
   *	one temp file then sends it to the output TAR archive.
   *
   *	The tar format also includes a 'restore.sql' script which is there for
   *	the benefit of humans. This script is never used by pg_restore.
   *
   *	NOTE: If you untar the created 'tar' file, the resulting files are
   *	compatible with the 'directory' format. Please keep the two formats in
   *	sync.
   *
   *	See the headers to pg_backup_directory & pg_restore for more details.
   *
   * Copyright (c) 2000, Philip Warner
   *		Rights are granted to use this software in any way so long
   *		as this notice is not removed.
   *
   *	The author is not responsible for loss or damages that may
   *	result from it's use.
   *
   *
   * IDENTIFICATION
   *		src/bin/pg_dump/pg_backup_tar.c
   *
   *-------------------------------------------------------------------------
   */
  
  #include "pg_backup.h"
  #include "pg_backup_archiver.h"
  #include "pg_backup_tar.h"
  #include "pg_backup_utils.h"
  #include "parallel.h"
  #include "pgtar.h"
  
  #include <sys/stat.h>
  #include <ctype.h>
  #include <limits.h>
  #include <unistd.h>
  
  static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
  static void _StartData(ArchiveHandle *AH, TocEntry *te);
  static size_t _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
  static void _EndData(ArchiveHandle *AH, TocEntry *te);
  static int	_WriteByte(ArchiveHandle *AH, const int i);
  static int	_ReadByte(ArchiveHandle *);
  static size_t _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
  static size_t _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
  static void _CloseArchive(ArchiveHandle *AH);
  static void _PrintTocData(ArchiveHandle *AH, TocEntry *te, RestoreOptions *ropt);
  static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
  static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
  static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
  
  static void _StartBlobs(ArchiveHandle *AH, TocEntry *te);
  static void _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
  static void _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
  static void _EndBlobs(ArchiveHandle *AH, TocEntry *te);
  
  #define K_STD_BUF_SIZE 1024
  
  
  typedef struct
  {
  #ifdef HAVE_LIBZ
  	gzFile		zFH;
  #else
  	FILE	   *zFH;
  #endif
  	FILE	   *nFH;
  	FILE	   *tarFH;
  	FILE	   *tmpFH;
  	char	   *targetFile;
  	char		mode;
  	pgoff_t		pos;
  	pgoff_t		fileLen;
  	ArchiveHandle *AH;
  } TAR_MEMBER;
  
  /*
   * Maximum file size for a tar member: The limit inherent in the
   * format is 2^33-1 bytes (nearly 8 GB).  But we don't want to exceed
   * what we can represent in pgoff_t.
   */
  #define MAX_TAR_MEMBER_FILELEN (((int64) 1 << Min(33, sizeof(pgoff_t)*8 - 1)) - 1)
  
  typedef struct
  {
  	int			hasSeek;
  	pgoff_t		filePos;
  	TAR_MEMBER *blobToc;
  	FILE	   *tarFH;
  	pgoff_t		tarFHpos;
  	pgoff_t		tarNextMember;
  	TAR_MEMBER *FH;
  	int			isSpecialScript;
  	TAR_MEMBER *scriptTH;
  } lclContext;
  
  typedef struct
  {
  	TAR_MEMBER *TH;
  	char	   *filename;
  } lclTocEntry;
  
  /* translator: this is a module name */
  static const char *modulename = gettext_noop("tar archiver");
  
  static void _LoadBlobs(ArchiveHandle *AH, RestoreOptions *ropt);
  
  static TAR_MEMBER *tarOpen(ArchiveHandle *AH, const char *filename, char mode);
  static void tarClose(ArchiveHandle *AH, TAR_MEMBER *TH);
  
  #ifdef __NOT_USED__
  static char *tarGets(char *buf, size_t len, TAR_MEMBER *th);
  #endif
  static int	tarPrintf(ArchiveHandle *AH, TAR_MEMBER *th, const char *fmt,...) __attribute__((format(PG_PRINTF_ATTRIBUTE, 3, 4)));
  
  static void _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th);
  static TAR_MEMBER *_tarPositionTo(ArchiveHandle *AH, const char *filename);
  static size_t tarRead(void *buf, size_t len, TAR_MEMBER *th);
  static size_t tarWrite(const void *buf, size_t len, TAR_MEMBER *th);
  static void _tarWriteHeader(TAR_MEMBER *th);
  static int	_tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th);
  static size_t _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh);
  
  static size_t _scriptOut(ArchiveHandle *AH, const void *buf, size_t len);
  
  /*
   *	Initializer
   */
  void
  InitArchiveFmt_Tar(ArchiveHandle *AH)
  {
  	lclContext *ctx;
  
  	/* Assuming static functions, this can be copied for each format. */
  	AH->ArchiveEntryPtr = _ArchiveEntry;
  	AH->StartDataPtr = _StartData;
  	AH->WriteDataPtr = _WriteData;
  	AH->EndDataPtr = _EndData;
  	AH->WriteBytePtr = _WriteByte;
  	AH->ReadBytePtr = _ReadByte;
  	AH->WriteBufPtr = _WriteBuf;
  	AH->ReadBufPtr = _ReadBuf;
  	AH->ClosePtr = _CloseArchive;
  	AH->ReopenPtr = NULL;
  	AH->PrintTocDataPtr = _PrintTocData;
  	AH->ReadExtraTocPtr = _ReadExtraToc;
  	AH->WriteExtraTocPtr = _WriteExtraToc;
  	AH->PrintExtraTocPtr = _PrintExtraToc;
  
  	AH->StartBlobsPtr = _StartBlobs;
  	AH->StartBlobPtr = _StartBlob;
  	AH->EndBlobPtr = _EndBlob;
  	AH->EndBlobsPtr = _EndBlobs;
  	AH->ClonePtr = NULL;
  	AH->DeClonePtr = NULL;
  
  	AH->MasterStartParallelItemPtr = NULL;
  	AH->MasterEndParallelItemPtr = NULL;
  
  	AH->WorkerJobDumpPtr = NULL;
  	AH->WorkerJobRestorePtr = NULL;
  
  	/*
  	 * Set up some special context used in compressing data.
  	 */
  	ctx = (lclContext *) pg_malloc0(sizeof(lclContext));
  	AH->formatData = (void *) ctx;
  	ctx->filePos = 0;
  	ctx->isSpecialScript = 0;
  
  	/* Initialize LO buffering */
  	AH->lo_buf_size = LOBBUFSIZE;
  	AH->lo_buf = (void *) pg_malloc(LOBBUFSIZE);
  
  	/*
  	 * Now open the tar file, and load the TOC if we're in read mode.
  	 */
  	if (AH->mode == archModeWrite)
  	{
  		if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
  		{
  			ctx->tarFH = fopen(AH->fSpec, PG_BINARY_W);
  			if (ctx->tarFH == NULL)
  				exit_horribly(modulename,
  						   "could not open TOC file \"%s\" for output: %s\n",
  							  AH->fSpec, strerror(errno));
  		}
  		else
  		{
  			ctx->tarFH = stdout;
  			if (ctx->tarFH == NULL)
  				exit_horribly(modulename,
  							  "could not open TOC file for output: %s\n",
  							  strerror(errno));
  		}
  
  		ctx->tarFHpos = 0;
  
  		/*
  		 * Make unbuffered since we will dup() it, and the buffers screw each
  		 * other
  		 */
  		/* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
  
  		ctx->hasSeek = checkSeek(ctx->tarFH);
  
  		if (AH->compression < 0 || AH->compression > 9)
  			AH->compression = Z_DEFAULT_COMPRESSION;
  
  		/* Don't compress into tar files unless asked to do so */
  		if (AH->compression == Z_DEFAULT_COMPRESSION)
  			AH->compression = 0;
  
  		/*
  		 * We don't support compression because reading the files back is not
  		 * possible since gzdopen uses buffered IO which totally screws file
  		 * positioning.
  		 */
  		if (AH->compression != 0)
  			exit_horribly(modulename,
  					 "compression is not supported by tar archive format\n");
  	}
  	else
  	{							/* Read Mode */
  		if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
  		{
  			ctx->tarFH = fopen(AH->fSpec, PG_BINARY_R);
  			if (ctx->tarFH == NULL)
  				exit_horribly(modulename, "could not open TOC file \"%s\" for input: %s\n",
  							  AH->fSpec, strerror(errno));
  		}
  		else
  		{
  			ctx->tarFH = stdin;
  			if (ctx->tarFH == NULL)
  				exit_horribly(modulename, "could not open TOC file for input: %s\n",
  							  strerror(errno));
  		}
  
  		/*
  		 * Make unbuffered since we will dup() it, and the buffers screw each
  		 * other
  		 */
  		/* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
  
  		ctx->tarFHpos = 0;
  
  		ctx->hasSeek = checkSeek(ctx->tarFH);
  
  		/*
  		 * Forcibly unmark the header as read since we use the lookahead
  		 * buffer
  		 */
  		AH->readHeader = 0;
  
  		ctx->FH = (void *) tarOpen(AH, "toc.dat", 'r');
  		ReadHead(AH);
  		ReadToc(AH);
  		tarClose(AH, ctx->FH);	/* Nothing else in the file... */
  	}
  }
  
  /*
   * - Start a new TOC entry
   *	 Setup the output file name.
   */
  static void
  _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
  {
  	lclTocEntry *ctx;
  	char		fn[K_STD_BUF_SIZE];
  
  	ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
  	if (te->dataDumper != NULL)
  	{
  #ifdef HAVE_LIBZ
  		if (AH->compression == 0)
  			sprintf(fn, "%d.dat", te->dumpId);
  		else
  			sprintf(fn, "%d.dat.gz", te->dumpId);
  #else
  		sprintf(fn, "%d.dat", te->dumpId);
  #endif
  		ctx->filename = pg_strdup(fn);
  	}
  	else
  	{
  		ctx->filename = NULL;
  		ctx->TH = NULL;
  	}
  	te->formatData = (void *) ctx;
  }
  
  static void
  _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
  {
  	lclTocEntry *ctx = (lclTocEntry *) te->formatData;
  
  	if (ctx->filename)
  		WriteStr(AH, ctx->filename);
  	else
  		WriteStr(AH, "");
  }
  
  static void
  _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
  {
  	lclTocEntry *ctx = (lclTocEntry *) te->formatData;
  
  	if (ctx == NULL)
  	{
  		ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
  		te->formatData = (void *) ctx;
  	}
  
  	ctx->filename = ReadStr(AH);
  	if (strlen(ctx->filename) == 0)
  	{
  		free(ctx->filename);
  		ctx->filename = NULL;
  	}
  	ctx->TH = NULL;
  }
  
  static void
  _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
  {
  	lclTocEntry *ctx = (lclTocEntry *) te->formatData;
  
  	if (AH->public.verbose && ctx->filename != NULL)
  		ahprintf(AH, "-- File: %s\n", ctx->filename);
  }
  
  static void
  _StartData(ArchiveHandle *AH, TocEntry *te)
  {
  	lclTocEntry *tctx = (lclTocEntry *) te->formatData;
  
  	tctx->TH = tarOpen(AH, tctx->filename, 'w');
  }
  
  static TAR_MEMBER *
  tarOpen(ArchiveHandle *AH, const char *filename, char mode)
  {
  	lclContext *ctx = (lclContext *) AH->formatData;
  	TAR_MEMBER *tm;
  
  #ifdef HAVE_LIBZ
  	char		fmode[10];
  #endif
  
  	if (mode == 'r')
  	{
  		tm = _tarPositionTo(AH, filename);
  		if (!tm)				/* Not found */
  		{
  			if (filename)
  			{
  				/*
  				 * Couldn't find the requested file. Future: do SEEK(0) and
  				 * retry.
  				 */
  				exit_horribly(modulename, "could not find file \"%s\" in archive\n", filename);
  			}
  			else
  			{
  				/* Any file OK, none left, so return NULL */
  				return NULL;
  			}
  		}
  
  #ifdef HAVE_LIBZ
  
  		if (AH->compression == 0)
  			tm->nFH = ctx->tarFH;
  		else
  			exit_horribly(modulename, "compression is not supported by tar archive format\n");
  		/* tm->zFH = gzdopen(dup(fileno(ctx->tarFH)), "rb"); */
  #else
  		tm->nFH = ctx->tarFH;
  #endif
  	}
  	else
  	{
  		tm = pg_malloc0(sizeof(TAR_MEMBER));
  
  #ifndef WIN32
  		tm->tmpFH = tmpfile();
  #else
  
  		/*
  		 * On WIN32, tmpfile() generates a filename in the root directory,
  		 * which requires administrative permissions on certain systems. Loop
  		 * until we find a unique file name we can create.
  		 */
  		while (1)
  		{
  			char	   *name;
  			int			fd;
  
  			name = _tempnam(NULL, "pg_temp_");
  			if (name == NULL)
  				break;
  			fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY |
  					  O_TEMPORARY, S_IRUSR | S_IWUSR);
  			free(name);
  
  			if (fd != -1)		/* created a file */
  			{
  				tm->tmpFH = fdopen(fd, "w+b");
  				break;
  			}
  			else if (errno != EEXIST)	/* failure other than file exists */
  				break;
  		}
  #endif
  
  		if (tm->tmpFH == NULL)
  			exit_horribly(modulename, "could not generate temporary file name: %s\n", strerror(errno));
  
  #ifdef HAVE_LIBZ
  
  		if (AH->compression != 0)
  		{
  			sprintf(fmode, "wb%d", AH->compression);
  			tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
  			if (tm->zFH == NULL)
  				exit_horribly(modulename, "could not open temporary file\n");
  		}
  		else
  			tm->nFH = tm->tmpFH;
  #else
  
  		tm->nFH = tm->tmpFH;
  #endif
  
  		tm->AH = AH;
  		tm->targetFile = pg_strdup(filename);
  	}
  
  	tm->mode = mode;
  	tm->tarFH = ctx->tarFH;
  
  	return tm;
  }
  
  static void
  tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
  {
  	/*
  	 * Close the GZ file since we dup'd. This will flush the buffers.
  	 */
  	if (AH->compression != 0)
  		if (GZCLOSE(th->zFH) != 0)
  			exit_horribly(modulename, "could not close tar member\n");
  
  	if (th->mode == 'w')
  		_tarAddFile(AH, th);	/* This will close the temp file */
  
  	/*
  	 * else Nothing to do for normal read since we don't dup() normal file
  	 * handle, and we don't use temp files.
  	 */
  
  	if (th->targetFile)
  		free(th->targetFile);
  
  	th->nFH = NULL;
  	th->zFH = NULL;
  }
  
  #ifdef __NOT_USED__
  static char *
  tarGets(char *buf, size_t len, TAR_MEMBER *th)
  {
  	char	   *s;
  	size_t		cnt = 0;
  	char		c = ' ';
  	int			eof = 0;
  
  	/* Can't read past logical EOF */
  	if (len > (th->fileLen - th->pos))
  		len = th->fileLen - th->pos;
  
  	while (cnt < len && c != '\n')
  	{
  		if (_tarReadRaw(th->AH, &c, 1, th, NULL) <= 0)
  		{
  			eof = 1;
  			break;
  		}
  		buf[cnt++] = c;
  	}
  
  	if (eof && cnt == 0)
  		s = NULL;
  	else
  	{
  		buf[cnt++] = '\0';
  		s = buf;
  	}
  
  	if (s)
  	{
  		len = strlen(s);
  		th->pos += len;
  	}
  
  	return s;
  }
  #endif
  
  /*
   * Just read bytes from the archive. This is the low level read routine
   * that is used for ALL reads on a tar file.
   */
  static size_t
  _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh)
  {
  	lclContext *ctx = (lclContext *) AH->formatData;
  	size_t		avail;
  	size_t		used = 0;
  	size_t		res = 0;
  
  	avail = AH->lookaheadLen - AH->lookaheadPos;
  	if (avail > 0)
  	{
  		/* We have some lookahead bytes to use */
  		if (avail >= len)		/* Just use the lookahead buffer */
  			used = len;
  		else
  			used = avail;
  
  		/* Copy, and adjust buffer pos */
  		memcpy(buf, AH->lookahead + AH->lookaheadPos, used);
  		AH->lookaheadPos += used;
  
  		/* Adjust required length */
  		len -= used;
  	}
  
  	/* Read the file if len > 0 */
  	if (len > 0)
  	{
  		if (fh)
  			res = fread(&((char *) buf)[used], 1, len, fh);
  		else if (th)
  		{
  			if (th->zFH)
  				res = GZREAD(&((char *) buf)[used], 1, len, th->zFH);
  			else
  				res = fread(&((char *) buf)[used], 1, len, th->nFH);
  		}
  		else
  			exit_horribly(modulename, "internal error -- neither th nor fh specified in tarReadRaw()\n");
  	}
  
  	ctx->tarFHpos += res + used;
  
  	return (res + used);
  }
  
  static size_t
  tarRead(void *buf, size_t len, TAR_MEMBER *th)
  {
  	size_t		res;
  
  	if (th->pos + len > th->fileLen)
  		len = th->fileLen - th->pos;
  
  	if (len <= 0)
  		return 0;
  
  	res = _tarReadRaw(th->AH, buf, len, th, NULL);
  
  	th->pos += res;
  
  	return res;
  }
  
  static size_t
  tarWrite(const void *buf, size_t len, TAR_MEMBER *th)
  {
  	size_t		res;
  
  	if (th->zFH != NULL)
  		res = GZWRITE(buf, 1, len, th->zFH);
  	else
  		res = fwrite(buf, 1, len, th->nFH);
  
  	if (res != len)
  		exit_horribly(modulename,
  					"could not write to output file: %s\n", strerror(errno));
  
  	th->pos += res;
  	return res;
  }
  
  static size_t
  _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
  {
  	lclTocEntry *tctx = (lclTocEntry *) AH->currToc->formatData;
  
  	dLen = tarWrite(data, dLen, tctx->TH);
  
  	return dLen;
  }
  
  static void
  _EndData(ArchiveHandle *AH, TocEntry *te)
  {
  	lclTocEntry *tctx = (lclTocEntry *) te->formatData;
  
  	/* Close the file */
  	tarClose(AH, tctx->TH);
  	tctx->TH = NULL;
  }
  
  /*
   * Print data for a given file
   */
  static void
  _PrintFileData(ArchiveHandle *AH, char *filename, RestoreOptions *ropt)
  {
  	lclContext *ctx = (lclContext *) AH->formatData;
  	char		buf[4096];
  	size_t		cnt;
  	TAR_MEMBER *th;
  
  	if (!filename)
  		return;
  
  	th = tarOpen(AH, filename, 'r');
  	ctx->FH = th;
  
  	while ((cnt = tarRead(buf, 4095, th)) > 0)
  	{
  		buf[cnt] = '\0';
  		ahwrite(buf, 1, cnt, AH);
  	}
  
  	tarClose(AH, th);
  }
  
  
  /*
   * Print data for a given TOC entry
  */
  static void
  _PrintTocData(ArchiveHandle *AH, TocEntry *te, RestoreOptions *ropt)
  {
  	lclContext *ctx = (lclContext *) AH->formatData;
  	lclTocEntry *tctx = (lclTocEntry *) te->formatData;
  	int			pos1;
  
  	if (!tctx->filename)
  		return;
  
  	/*
  	 * If we're writing the special restore.sql script, emit a suitable
  	 * command to include each table's data from the corresponding file.
  	 *
  	 * In the COPY case this is a bit klugy because the regular COPY command
  	 * was already printed before we get control.
  	 */
  	if (ctx->isSpecialScript)
  	{
  		if (te->copyStmt)
  		{
  			/* Abort the COPY FROM stdin */
  			ahprintf(AH, "\\.\n");
  
  			/*
  			 * The COPY statement should look like "COPY ... FROM stdin;\n",
  			 * see dumpTableData().
  			 */
  			pos1 = (int) strlen(te->copyStmt) - 13;
  			if (pos1 < 6 || strncmp(te->copyStmt, "COPY ", 5) != 0 ||
  				strcmp(te->copyStmt + pos1, " FROM stdin;\n") != 0)
  				exit_horribly(modulename,
  							  "unexpected COPY statement syntax: \"%s\"\n",
  							  te->copyStmt);
  
  			/* Emit all but the FROM part ... */
  			ahwrite(te->copyStmt, 1, pos1, AH);
  			/* ... and insert modified FROM */
  			ahprintf(AH, " FROM '$$PATH$$/%s';\n\n", tctx->filename);
  		}
  		else
  		{
  			/* --inserts mode, no worries, just include the data file */
  			ahprintf(AH, "\\i $$PATH$$/%s\n\n", tctx->filename);
  		}
  
  		return;
  	}
  
  	if (strcmp(te->desc, "BLOBS") == 0)
  		_LoadBlobs(AH, ropt);
  	else
  		_PrintFileData(AH, tctx->filename, ropt);
  }
  
  static void
  _LoadBlobs(ArchiveHandle *AH, RestoreOptions *ropt)
  {
  	Oid			oid;
  	lclContext *ctx = (lclContext *) AH->formatData;
  	TAR_MEMBER *th;
  	size_t		cnt;
  	bool		foundBlob = false;
  	char		buf[4096];
  
  	StartRestoreBlobs(AH);
  
  	th = tarOpen(AH, NULL, 'r');	/* Open next file */
  	while (th != NULL)
  	{
  		ctx->FH = th;
  
  		if (strncmp(th->targetFile, "blob_", 5) == 0)
  		{
  			oid = atooid(&th->targetFile[5]);
  			if (oid != 0)
  			{
  				ahlog(AH, 1, "restoring large object with OID %u\n", oid);
  
  				StartRestoreBlob(AH, oid, ropt->dropSchema);
  
  				while ((cnt = tarRead(buf, 4095, th)) > 0)
  				{
  					buf[cnt] = '\0';
  					ahwrite(buf, 1, cnt, AH);
  				}
  				EndRestoreBlob(AH, oid);
  				foundBlob = true;
  			}
  			tarClose(AH, th);
  		}
  		else
  		{
  			tarClose(AH, th);
  
  			/*
  			 * Once we have found the first blob, stop at the first non-blob
  			 * entry (which will be 'blobs.toc').  This coding would eat all
  			 * the rest of the archive if there are no blobs ... but this
  			 * function shouldn't be called at all in that case.
  			 */
  			if (foundBlob)
  				break;
  		}
  
  		th = tarOpen(AH, NULL, 'r');
  	}
  	EndRestoreBlobs(AH);
  }
  
  
  static int
  _WriteByte(ArchiveHandle *AH, const int i)
  {
  	lclContext *ctx = (lclContext *) AH->formatData;
  	int			res;
  	char		b = i;			/* Avoid endian problems */
  
  	res = tarWrite(&b, 1, ctx->FH);
  	if (res != EOF)
  		ctx->filePos += res;
  	return res;
  }
  
  static int
  _ReadByte(ArchiveHandle *AH)
  {
  	lclContext *ctx = (lclContext *) AH->formatData;
  	size_t		res;
  	unsigned char c;
  
  	res = tarRead(&c, 1, ctx->FH);
  	if (res != 1)
  		exit_horribly(modulename, "unexpected end of file\n");
  	ctx->filePos += 1;
  	return c;
  }
  
  static size_t
  _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
  {
  	lclContext *ctx = (lclContext *) AH->formatData;
  	size_t		res;
  
  	res = tarWrite(buf, len, ctx->FH);
  	ctx->filePos += res;
  	return res;
  }
  
  static size_t
  _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
  {
  	lclContext *ctx = (lclContext *) AH->formatData;
  	size_t		res;
  
  	res = tarRead(buf, len, ctx->FH);
  	ctx->filePos += res;
  	return res;
  }
  
  static void
  _CloseArchive(ArchiveHandle *AH)
  {
  	lclContext *ctx = (lclContext *) AH->formatData;
  	TAR_MEMBER *th;
  	RestoreOptions *ropt;
  	RestoreOptions *savRopt;
  	int			savVerbose,
  				i;
  
  	if (AH->mode == archModeWrite)
  	{
  		/*
  		 * Write the Header & TOC to the archive FIRST
  		 */
  		th = tarOpen(AH, "toc.dat", 'w');
  		ctx->FH = th;
  		WriteHead(AH);
  		WriteToc(AH);
  		tarClose(AH, th);		/* Not needed any more */
  
  		/*
  		 * Now send the data (tables & blobs)
  		 */
  		WriteDataChunks(AH, NULL);
  
  		/*
  		 * Now this format wants to append a script which does a full restore
  		 * if the files have been extracted.
  		 */
  		th = tarOpen(AH, "restore.sql", 'w');
  
  		tarPrintf(AH, th, "--\n"
  				  "-- NOTE:\n"
  				  "--\n"
  				  "-- File paths need to be edited. Search for $$PATH$$ and\n"
  				  "-- replace it with the path to the directory containing\n"
  				  "-- the extracted data files.\n"
  				  "--\n");
  
  		AH->CustomOutPtr = _scriptOut;
  
  		ctx->isSpecialScript = 1;
  		ctx->scriptTH = th;
  
  		ropt = NewRestoreOptions();
  		memcpy(ropt, AH->ropt, sizeof(RestoreOptions));
  		ropt->filename = NULL;
  		ropt->dropSchema = 1;
  		ropt->compression = 0;
  		ropt->superuser = NULL;
  		ropt->suppressDumpWarnings = true;
  
  		savRopt = AH->ropt;
  		AH->ropt = ropt;
  
  		savVerbose = AH->public.verbose;
  		AH->public.verbose = 0;
  
  		RestoreArchive((Archive *) AH);
  
  		AH->ropt = savRopt;
  		AH->public.verbose = savVerbose;
  
  		tarClose(AH, th);
  
  		ctx->isSpecialScript = 0;
  
  		/*
  		 * EOF marker for tar files is two blocks of NULLs.
  		 */
  		for (i = 0; i < 512 * 2; i++)
  		{
  			if (fputc(0, ctx->tarFH) == EOF)
  				exit_horribly(modulename,
  					   "could not write null block at end of tar archive\n");
  		}
  	}
  
  	AH->FH = NULL;
  }
  
  static size_t
  _scriptOut(ArchiveHandle *AH, const void *buf, size_t len)
  {
  	lclContext *ctx = (lclContext *) AH->formatData;
  
  	return tarWrite(buf, len, ctx->scriptTH);
  }
  
  /*
   * BLOB support
   */
  
  /*
   * Called by the archiver when starting to save all BLOB DATA (not schema).
   * This routine should save whatever format-specific information is needed
   * to read the BLOBs back into memory.
   *
   * It is called just prior to the dumper's DataDumper routine.
   *
   * Optional, but strongly recommended.
   *
   */
  static void
  _StartBlobs(ArchiveHandle *AH, TocEntry *te)
  {
  	lclContext *ctx = (lclContext *) AH->formatData;
  	char		fname[K_STD_BUF_SIZE];
  
  	sprintf(fname, "blobs.toc");
  	ctx->blobToc = tarOpen(AH, fname, 'w');
  }
  
  /*
   * Called by the archiver when the dumper calls StartBlob.
   *
   * Mandatory.
   *
   * Must save the passed OID for retrieval at restore-time.
   */
  static void
  _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
  {
  	lclContext *ctx = (lclContext *) AH->formatData;
  	lclTocEntry *tctx = (lclTocEntry *) te->formatData;
  	char		fname[255];
  	char	   *sfx;
  
  	if (oid == 0)
  		exit_horribly(modulename, "invalid OID for large object (%u)\n", oid);
  
  	if (AH->compression != 0)
  		sfx = ".gz";
  	else
  		sfx = "";
  
  	sprintf(fname, "blob_%u.dat%s", oid, sfx);
  
  	tarPrintf(AH, ctx->blobToc, "%u %s\n", oid, fname);
  
  	tctx->TH = tarOpen(AH, fname, 'w');
  }
  
  /*
   * Called by the archiver when the dumper calls EndBlob.
   *
   * Optional.
   *
   */
  static void
  _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
  {
  	lclTocEntry *tctx = (lclTocEntry *) te->formatData;
  
  	tarClose(AH, tctx->TH);
  }
  
  /*
   * Called by the archiver when finishing saving all BLOB DATA.
   *
   * Optional.
   *
   */
  static void
  _EndBlobs(ArchiveHandle *AH, TocEntry *te)
  {
  	lclContext *ctx = (lclContext *) AH->formatData;
  
  	/* Write out a fake zero OID to mark end-of-blobs. */
  	/* WriteInt(AH, 0); */
  
  	tarClose(AH, ctx->blobToc);
  }
  
  
  
  /*------------
   * TAR Support
   *------------
   */
  
  static int
  tarPrintf(ArchiveHandle *AH, TAR_MEMBER *th, const char *fmt,...)
  {
  	char	   *p;
  	size_t		len = 128;		/* initial assumption about buffer size */
  	size_t		cnt;
  
  	for (;;)
  	{
  		va_list		args;
  
  		/* Allocate work buffer. */
  		p = (char *) pg_malloc(len);
  
  		/* Try to format the data. */
  		va_start(args, fmt);
  		cnt = pvsnprintf(p, len, fmt, args);
  		va_end(args);
  
  		if (cnt < len)
  			break;				/* success */
  
  		/* Release buffer and loop around to try again with larger len. */
  		free(p);
  		len = cnt;
  	}
  
  	cnt = tarWrite(p, cnt, th);
  	free(p);
  	return (int) cnt;
  }
  
  bool
  isValidTarHeader(char *header)
  {
  	int			sum;
  	int			chk = tarChecksum(header);
  
  	sscanf(&header[148], "%8o", &sum);
  
  	if (sum != chk)
  		return false;
  
  	/* POSIX tar format */
  	if (memcmp(&header[257], "ustar\0", 6) == 0 &&
  		memcmp(&header[263], "00", 2) == 0)
  		return true;
  	/* GNU tar format */
  	if (memcmp(&header[257], "ustar  \0", 8) == 0)
  		return true;
  	/* not-quite-POSIX format written by pre-9.3 pg_dump */
  	if (memcmp(&header[257], "ustar00\0", 8) == 0)
  		return true;
  
  	return false;
  }
  
  /* Given the member, write the TAR header & copy the file */
  static void
  _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th)
  {
  	lclContext *ctx = (lclContext *) AH->formatData;
  	FILE	   *tmp = th->tmpFH;	/* Grab it for convenience */
  	char		buf[32768];
  	size_t		cnt;
  	pgoff_t		len = 0;
  	size_t		res;
  	size_t		i,
  				pad;
  
  	/*
  	 * Find file len & go back to start.
  	 */
  	fseeko(tmp, 0, SEEK_END);
  	th->fileLen = ftello(tmp);
  	fseeko(tmp, 0, SEEK_SET);
  
  	/*
  	 * Some compilers will throw a warning knowing this test can never be true
  	 * because pgoff_t can't exceed the compared maximum on their platform.
  	 */
  	if (th->fileLen > MAX_TAR_MEMBER_FILELEN)
  		exit_horribly(modulename, "archive member too large for tar format\n");
  
  	_tarWriteHeader(th);
  
  	while ((cnt = fread(buf, 1, sizeof(buf), tmp)) > 0)
  	{
  		res = fwrite(buf, 1, cnt, th->tarFH);
  		if (res != cnt)
  			exit_horribly(modulename,
  						  "could not write to output file: %s\n",
  						  strerror(errno));
  		len += res;
  	}
  
  	if (fclose(tmp) != 0)		/* This *should* delete it... */
  		exit_horribly(modulename, "could not close temporary file: %s\n",
  					  strerror(errno));
  
  	if (len != th->fileLen)
  	{
  		char		buf1[32],
  					buf2[32];
  
  		snprintf(buf1, sizeof(buf1), INT64_FORMAT, (int64) len);
  		snprintf(buf2, sizeof(buf2), INT64_FORMAT, (int64) th->fileLen);
  		exit_horribly(modulename, "actual file length (%s) does not match expected (%s)\n",
  					  buf1, buf2);
  	}
  
  	pad = ((len + 511) & ~511) - len;
  	for (i = 0; i < pad; i++)
  	{
  		if (fputc('\0', th->tarFH) == EOF)
  			exit_horribly(modulename, "could not output padding at end of tar member\n");
  	}
  
  	ctx->tarFHpos += len + pad;
  }
  
  /* Locate the file in the archive, read header and position to data */
  static TAR_MEMBER *
  _tarPositionTo(ArchiveHandle *AH, const char *filename)
  {
  	lclContext *ctx = (lclContext *) AH->formatData;
  	TAR_MEMBER *th = pg_malloc0(sizeof(TAR_MEMBER));
  	char		c;
  	char		header[512];
  	size_t		i,
  				len,
  				blks;
  	int			id;
  
  	th->AH = AH;
  
  	/* Go to end of current file, if any */
  	if (ctx->tarFHpos != 0)
  	{
  		char		buf1[100],
  					buf2[100];
  
  		snprintf(buf1, sizeof(buf1), INT64_FORMAT, (int64) ctx->tarFHpos);
  		snprintf(buf2, sizeof(buf2), INT64_FORMAT, (int64) ctx->tarNextMember);
  		ahlog(AH, 4, "moving from position %s to next member at file position %s\n",
  			  buf1, buf2);
  
  		while (ctx->tarFHpos < ctx->tarNextMember)
  			_tarReadRaw(AH, &c, 1, NULL, ctx->tarFH);
  	}
  
  	{
  		char		buf[100];
  
  		snprintf(buf, sizeof(buf), INT64_FORMAT, (int64) ctx->tarFHpos);
  		ahlog(AH, 4, "now at file position %s\n", buf);
  	}
  
  	/* We are at the start of the file, or at the next member */
  
  	/* Get the header */
  	if (!_tarGetHeader(AH, th))
  	{
  		if (filename)
  			exit_horribly(modulename, "could not find header for file \"%s\" in tar archive\n", filename);
  		else
  		{
  			/*
  			 * We're just scanning the archive for the next file, so return
  			 * null
  			 */
  			free(th);
  			return NULL;
  		}
  	}
  
  	while (filename != NULL && strcmp(th->targetFile, filename) != 0)
  	{
  		ahlog(AH, 4, "skipping tar member %s\n", th->targetFile);
  
  		id = atoi(th->targetFile);
  		if ((TocIDRequired(AH, id) & REQ_DATA) != 0)
  			exit_horribly(modulename, "restoring data out of order is not supported in this archive format: "
  						  "\"%s\" is required, but comes before \"%s\" in the archive file.\n",
  						  th->targetFile, filename);
  
  		/* Header doesn't match, so read to next header */
  		len = ((th->fileLen + 511) & ~511);		/* Padded length */
  		blks = len >> 9;		/* # of 512 byte blocks */
  
  		for (i = 0; i < blks; i++)
  			_tarReadRaw(AH, &header[0], 512, NULL, ctx->tarFH);
  
  		if (!_tarGetHeader(AH, th))
  			exit_horribly(modulename, "could not find header for file \"%s\" in tar archive\n", filename);
  	}
  
  	ctx->tarNextMember = ctx->tarFHpos + ((th->fileLen + 511) & ~511);
  	th->pos = 0;
  
  	return th;
  }
  
  /* Read & verify a header */
  static int
  _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th)
  {
  	lclContext *ctx = (lclContext *) AH->formatData;
  	char		h[512];
  	char		tag[100];
  	int			sum,
  				chk;
  	size_t		len;
  	unsigned long ullen;
  	pgoff_t		hPos;
  	bool		gotBlock = false;
  
  	while (!gotBlock)
  	{
  #if 0
  		if (ftello(ctx->tarFH) != ctx->tarFHpos)
  		{
  			char		buf1[100],
  						buf2[100];
  
  			snprintf(buf1, sizeof(buf1), INT64_FORMAT, (int64) ftello(ctx->tarFH));
  			snprintf(buf2, sizeof(buf2), INT64_FORMAT, (int64) ftello(ctx->tarFHpos));
  			exit_horribly(modulename,
  			  "mismatch in actual vs. predicted file position (%s vs. %s)\n",
  						  buf1, buf2);
  		}
  #endif
  
  		/* Save the pos for reporting purposes */
  		hPos = ctx->tarFHpos;
  
  		/* Read a 512 byte block, return EOF, exit if short */
  		len = _tarReadRaw(AH, h, 512, NULL, ctx->tarFH);
  		if (len == 0)			/* EOF */
  			return 0;
  
  		if (len != 512)
  			exit_horribly(modulename,
  						  ngettext("incomplete tar header found (%lu byte)\n",
  								 "incomplete tar header found (%lu bytes)\n",
  								   len),
  						  (unsigned long) len);
  
  		/* Calc checksum */
  		chk = tarChecksum(h);
  		sscanf(&h[148], "%8o", &sum);
  
  		/*
  		 * If the checksum failed, see if it is a null block. If so, silently
  		 * continue to the next block.
  		 */
  		if (chk == sum)
  			gotBlock = true;
  		else
  		{
  			int			i;
  
  			for (i = 0; i < 512; i++)
  			{
  				if (h[i] != 0)
  				{
  					gotBlock = true;
  					break;
  				}
  			}
  		}
  	}
  
  	sscanf(&h[0], "%99s", tag);
  	sscanf(&h[124], "%12lo", &ullen);
  	len = (size_t) ullen;
  
  	{
  		char		buf[100];
  
  		snprintf(buf, sizeof(buf), INT64_FORMAT, (int64) hPos);
  		ahlog(AH, 3, "TOC Entry %s at %s (length %lu, checksum %d)\n",
  			  tag, buf, (unsigned long) len, sum);
  	}
  
  	if (chk != sum)
  	{
  		char		buf[100];
  
  		snprintf(buf, sizeof(buf), INT64_FORMAT, (int64) ftello(ctx->tarFH));
  		exit_horribly(modulename,
  					  "corrupt tar header found in %s "
  					  "(expected %d, computed %d) file position %s\n",
  					  tag, sum, chk, buf);
  	}
  
  	th->targetFile = pg_strdup(tag);
  	th->fileLen = len;
  
  	return 1;
  }
  
  
  static void
  _tarWriteHeader(TAR_MEMBER *th)
  {
  	char		h[512];
  
  	tarCreateHeader(h, th->targetFile, NULL, th->fileLen, 0600, 04000, 02000, time(NULL));
  
  	/* Now write the completed header. */
  	if (fwrite(h, 1, 512, th->tarFH) != 512)
  		exit_horribly(modulename, "could not write to output file: %s\n", strerror(errno));
  }
  