Date: Sun, 11 Apr 1999 14:29:53 +0200
Message-Id: <199904111229.OAA07355@servalys.hobby.nl>
From: Wybo Dekker <wybo@servalys.hobby.nl>
To: matthew@venux.net
CC: pgsql-interfaces@postgreSQL.org
In-reply-to: <4.1.19990410172612.009943e0@mail.venux.net> (message from
	Matthew Hagerty on Sat, 10 Apr 1999 17:36:29 -0400)
Subject: Re: [INTERFACES] Fastest import?
References: <4.1.19990410172612.009943e0@mail.venux.net>


Matthew Hagerty wrote:
> 
> Greetings,
> 
> What is the fastest way to import data from an ASCII file?  I have ~17,000
> records in a .DBF file that I need to get into my pg6.4.2 database.  I have
> written a program that reads the .DBF file, massages the data and writes it
> out to a text file, one record per line.

I do that straight under linux by reading the .dbf file with a small
C-program which writes the data out in tab-separated records. Those
can be read with the copy from command:

/* __________ db2u __________
   convert dbase file
   Usage:
   db2u < dbase-file.dbf > unix_database
    or
   db2u dbase-file.dbf > unix_database
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#define MAXFIELDS 100 /* maximum nr of fields */

void main(int argc, char *argv[])
{ char *p,*q,*r;
  typedef char string11[12];
  FILE *in;
  int i,j,n,field_pos[MAXFIELDS+1],nfield=0,pos,lrec;
  char *record,field_type[MAXFIELDS];
  string11 field_name[MAXFIELDS];
  struct
  { char  version;    /* dBase version */
    char  datum[3];   /* date of last update YYMMDD */
    long  nrec;       /* nr of records in th database file */
    short nhead;      /* nr of bytes in the header */
    short lrec;       /* nr of bytes in the record */
    char  dum1[2];
    char  transact;   /* flag for incomplete transaction */
    char  encrypt;    /* encryption flag */
    char  lan[12];    /* reserved for network applications */
    char  mdx;        /* flag for presence of mdx-file */
    char  dum2[3];
  } header;

  struct
  { char  name[11];         /* fieldname zero filled */
    char  type;             /* field type (C, D, F, L, M, N) */
    char  dum1[4];
    unsigned char  length;  /* field length in binary */
    char  ndec;             /* field decimal count in binary */
    char  dum2[2];
    char  workid;           /* work area ID */
    char  dum3[11];
  } field;

  switch(argc)
  { case 1:   in=stdin;
              break;
    case 2:   if ((in = fopen(argv[1],"rb")) == NULL)
              { fprintf(stderr,"Input file %s not found\n",argv[1]);
                exit(1);
              }
              break;
    default:  fprintf(stderr,"Usage: %s dbase_file > unix_database\n",argv[0]);
              fprintf(stderr,"Or: cat dbase_file | %s > unix_database\n",argv[0]);
              exit(1);
  }
  fread(&header,32,1,in);
  n=header.lrec;
  pos=1;
  while (n>1)
  { if (nfield==MAXFIELDS)
    { fprintf(stderr,"Too many fields\n");
      exit(1);
    }
    fread(&field,32,1,in);
    strcpy(field_name[nfield],field.name);
    field_type[nfield]=field.type;
    field_pos[nfield++]=pos;
    pos+=(int)field.length;
    n-=(int)field.length;
  }
  for (i=0;i<nfield;i++) printf("!%d-%s%c",i+1,field_name[i],i==nfield-1?'\n':'\t');
  for (i=0;i<nfield;i++) printf(   "!%s%c",    field_name[i],i==nfield-1?'\n':'\t');
  field_pos[nfield]=pos; /* one extra for length calculation of last field */
  fread(&i,1,1,in); /*read field terminator */
  if (i!=0xD)
  { fprintf(stderr,"Illegal field terminator; more than %d fields?\n",MAXFIELDS);
    exit(1);
  }
  lrec=header.lrec;
  record=malloc(lrec);
  for (i=0;i<header.nrec;i++)
  { if (fread(record,lrec,1,in)!=1)
    { fprintf(stderr,"Error reading record %d\n",i+1);
      exit(1);
    }
    if (*record!='*')  /* if not a deleted record */
    for (j=0;j<nfield;j++)
    {
      p=record+field_pos[j];     		/* first char of the field */
      q=record+field_pos[j+1]-1; 		/* last char of the field */
      while (*p==' ' && p<=q) p++;		/* remove leading spaces */
      while (*q==' ' && p<=q) *(q--)='\0';	/* remove trailing spaces */
      for (r=p;r<=q;r++) putchar(*r);
      putchar(j==nfield-1?'\n':'\t');
    }
  }
}


-- 
Hartelijke groet, Wybo Dekker
___________________Servalys Analytical Chemistry Services__________________
wybo@servalys.hobby.nl | Deilsedijk 60                 | tel +31-345-652164
www.hobby.nl/~servalys | 4158 CH Deil, The Netherlands | fax
+31-345-652383