/*
   "db_files.c" - part of "RAGMAAN, een anagrammengenerator" 

   Copyright (C) 1999 Raymond Zandbergen

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

   The maintainer of this program is Raymond Zandbergen (ray@wirehub.nl)
 */

#include <stdio.h>
#include <ctype.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include "ragmaan.h"
#include "hash_func.h"
#include "search.h"
#include "db_files.h"

/*#define       rm_isprint(c) (isprint((c)) || (c) == (char)145) */
#define	rm_isprint(c) (isprint((c)))

/* divide the progress bar while loading */

#define	DB_WORD_COUNT_TIME	0.20
#define	DB_HISTOGRAM_TIME	0.20
#define	DB_HASH_TIME		0.60
#ifndef	GZIP_EXE
#define	GZIP_EXE	"/bin/gzip"
#endif

static char gzip_cmd[MAX_WORD_LENGTH] = "";
static char gzip_view_cmd[MAX_WORD_LENGTH] = "";

/*************************************************************************/
/**     fills GLOBALS gzip_cmd & gzip_view with appropriate strings     **/
/*************************************************************************/
static void gzip_pipes (void);

char *
read_words_from_file (const char *file_name, size_t * db_size)
{
  FILE *fp;
  char *buf;
  char *p;
  size_t alot;
  int nof_blocks;
  int n;
  int use_gzip;

/*************************************************************************/
/**     read words from file using block reads                          **/
/*************************************************************************/

#ifndef	NDEBUG
  printf ("reading from '%s'\n", file_name);
#endif

  gzip_pipes ();
  if (strlen (file_name) > MAX_WORD_LENGTH - strlen (gzip_cmd) - 2)
    return 0;

  p = strrchr (file_name, '.');

  use_gzip = p && (strcmp (p, ".gz") == 0 || strcmp (p, ".zip") == 0);

  if (use_gzip)
    {
/*************************************************************************/
/**     first, see if gzip can determine the uncompressed file size     **/
/**     by running 'gzip -ldq <filename>'                               **/
/*************************************************************************/

      char pipe_cmd[MAX_WORD_LENGTH];
      int dummy1, dummy2;

      strcpy (pipe_cmd, gzip_view_cmd);
      strcat (pipe_cmd, file_name);

      fp = popen (pipe_cmd, "r");
      if (!fp)
	{

	  perror ("popen");
	  return 0;
	}
      fscanf (fp, "%d %d", &dummy1, &dummy2);	/* no need to use fgets here */
      fclose (fp);

/*************************************************************************/
/**     reopen pipe to read the compressed file                         **/
/*************************************************************************/

      strcpy (pipe_cmd, gzip_cmd);
      strcat (pipe_cmd, file_name);

      fp = popen (pipe_cmd, "r");

      if (!fp)
	{

	  perror ("popen");
	  return 0;
	}

      if (dummy2 > 0)
	{
	  alot = (size_t) dummy2;	/* got file size! */
	}
      else
	{

/*************************************************************************/
/**     gzip couldn't get file size                                     **/
/**     determine length by reading & discarding - URGH!                **/
/*************************************************************************/

	  alot = 0;
	  while (!feof (fp))
	    {
	      getc (fp);
	      alot++;
	    }
	  if (alot)
	    alot--;
	  pclose (fp);
	  fp = popen (pipe_cmd, "r");
	}
    }
  else
    {

/*************************************************************************/
/**     not a compressed file,                                          **/
/**     determine length with fseek                                     **/
/*************************************************************************/

      fp = fopen (file_name, "r");
      if (!fp)
	{
	  perror ("fopen");
	  return 0;
	}
      fseek (fp, 0, SEEK_END);
      alot = ftell (fp);
      rewind (fp);
    }

  if (alot == 0)
    return 0;

/*************************************************************************/
/**     perform block reads                                             **/
/*************************************************************************/

  buf = malloc (alot);

  if (!buf)
    {

      fputs ("FATAL: out of memory\n", stderr);
      return 0;
    }

  nof_blocks = alot / BLOCK_SIZE;
  if (BLOCK_SIZE * nof_blocks < alot)
    nof_blocks++;

  n = fread (buf, BLOCK_SIZE, nof_blocks, fp);

#ifndef	NDEBUG
  printf ("%u full blocks / %u bytes read\n", n, alot);
#endif

  if (use_gzip)
    pclose (fp);
  else
    fclose (fp);

  *db_size = alot;
  return buf;
}

int
create_hash (HASH_TABLE_T * ht, char *words, size_t db_size,
	     progress_callback_t pc)
{
  int n;
  int i, j;
  int in_word, buf_offset;

  int word_length_histogram[MAX_WORD_LENGTH];
  int hash2;
  double inv_nof_words;
  double inv_db_size;

/*************************************************************************/
/**     count words                                                     **/
/*************************************************************************/

  ht->nof_words = 0;
  ht->dictionary_start = words;
  in_word = rm_isprint (words[0]);
  inv_db_size = 1.0 / db_size;
  for (i = 0; i < db_size; i++)
    {

      if (words[i] == (char) 145)
	words[i] = '\'';
      if (rm_isprint (words[i]))
	in_word = 1;
      if (!rm_isprint (words[i]) && in_word)
	{
	  ht->nof_words++;
	  in_word = 0;
	}
      if (i % (128 * 1024) == 0 && pc)
	{
	  pc ((int) 100 * (inv_db_size * i * DB_WORD_COUNT_TIME));
	}
    }

#ifndef	NDEBUG
  printf ("%d words indexed\n", ht->nof_words);
#endif

  ht->words = malloc (ht->nof_words * sizeof (char *));
  inv_nof_words = 1.0 / ht->nof_words;

  if (!ht->words)
    {
      fprintf (stderr, "FATAL: out of memory\n");
      return -1;
    }

/*************************************************************************/
/**     delimit strings / build histogram of word lengths               **/
/*************************************************************************/

  buf_offset = 0;
  for (i = 0; i < MAX_WORD_LENGTH; i++)
    word_length_histogram[i] = 0;
  for (i = 0; i < ht->nof_words; i++)
    {
      int word_length = 0;

      ht->words[i] = words + buf_offset;
      for (j = buf_offset; rm_isprint (words[j]); j++)
	{
	  if (isalnum (words[j]))
	    word_length++;
	}
      words[j] = 0;

      word_length_histogram[word_length]++;

      if (i < ht->nof_words - 1)
	for (++j; !rm_isprint (words[j]); j++);
      buf_offset = j;
      if (i % (8 * 1024) == 0 && pc)
	{
	  pc ((int) 100 *
	      (i * inv_nof_words * DB_HISTOGRAM_TIME + DB_WORD_COUNT_TIME));
	}
    }

  ht->max_observed_word_length = 0;
  for (i = 0; i < MAX_WORD_LENGTH; i++)
    {

      if (word_length_histogram[i])
	{

	  if (i > ht->max_observed_word_length)
	    {

	      ht->max_observed_word_length = i;
	      if (i >= MAX_WORD_LENGTH)
		{
		  fprintf (stderr, "OOPS - I don't know how to handle words of "
			   "%d characters\n", i);
		  exit (1);
		}
	    }

	}
    }

/*************************************************************************/
/**     setup hash table                                                **/
/*************************************************************************/

  n = ht->max_observed_word_length + 1;

  ht->hash2_table_key = malloc (n * sizeof (int *));
  ht->hash2_table_string = malloc (n * sizeof (char **));
  ht->hash2_table_length = malloc (n * sizeof (int));

  for (i = 0; i <= ht->max_observed_word_length; i++)
    {

      j = word_length_histogram[i];

/*
   printf("reserving %d hash entries for words of length %d\n",
   j,i);
 */

      ht->hash2_table_key[i] = malloc (j * sizeof (int));
      ht->hash2_table_string[i] = malloc (j * sizeof (char *));
      ht->hash2_table_length[i] = 0;
    }

#ifndef	NDEBUG
  puts ("creating hash tables");
#endif

  for (i = 0; i < ht->nof_words; i++)
    {

      if (i % (16 * 1024) == 0 && pc)
	{
	  pc ((int) 100 * (i * inv_nof_words * DB_HASH_TIME +
			   DB_HISTOGRAM_TIME + DB_WORD_COUNT_TIME));
	}

      n = rm_str_real_len (ht->words[i]);
      assert (n <= ht->max_observed_word_length);

      make_hash2 (ht->words[i], &hash2);

      if (hash2 == 0)
	{
	  return -1;		/* FATAL ERROR! */
	}

      j = ht->hash2_table_length[n]++;

#ifndef	NDEBUG
      if (j >= word_length_histogram[n])
	{

	  printf ("too many words of length %d (j = %d)\n", n, j);
	}
#endif

      ht->hash2_table_key[n][j] = hash2;
      ht->hash2_table_string[n][j] = ht->words[i];
    }

#ifndef	NDEBUG
  puts ("");
#endif
  return 0;
}

void
free_hash_table (HASH_TABLE_T * ht)
{
  int i;

  for (i = 0; i <= ht->max_observed_word_length; i++)
    {

      free (ht->hash2_table_key[i]);
      free (ht->hash2_table_string[i]);
    }
  free (ht->hash2_table_length);
  free (ht->hash2_table_key);
  free (ht->hash2_table_string);
  free (ht->words);
}

static void 
gzip_pipes (void)
{
  char *p;

  if (*gzip_cmd)
    return;

  p = getenv ("RM_GZIP_EXE");
  if (p)
    {
      strcpy (gzip_cmd, p);
    }
  else
    {
      fputs ("warning: env var RM_GZIP_EXE undefined - "
	     "assuming " GZIP_EXE "\n", stderr);
      strcpy (gzip_cmd, GZIP_EXE);
    }
  strcpy (gzip_view_cmd, gzip_cmd);
  strcat (gzip_cmd, " -cdq ");
  strcat (gzip_view_cmd, " -ldq ");
}