/*
    ragmaan/db_files.c            (C) 2003 Raymond (zandbergen@home.nl)

    "ragmaan" is an anagram generator

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#define NDEBUG
#include <assert.h>
#ifdef HAVE_CONFIG_H
#include <glib.h>
#include "config.h"
#endif
#include "ragmaan.h"
#include "hash_func.h"
#include "search.h"
#include "db_files.h"

#define MAX_WORD_LENGTH (120)

/* divide the progress bar while loading */

#define	DB_WORD_COUNT_TIME	0.60
#define	DB_HISTOGRAM_TIME	0.20
#define	DB_HASH_TIME		0.20

gchar *
read_words_from_file (const gchar * file_name, size_t * db_size,
		      void (pc) (void *, gdouble), void *pcdata)
{
  FILE *fp;
  gchar *buf;
  gchar *p;
  size_t alot;
  gint nof_blocks;
  gint n, block_count;
  gint use_gzip, use_bzip2;
  gdouble inv_nof_blocks;

/*************************************************************************/
/**     read words from file using block reads                          **/
/*************************************************************************/

#ifndef	NDEBUG
  printf ("reading from '%s'\n", file_name);
#endif

  p = strrchr (file_name, '.');

  use_gzip = p && (g_ascii_strcasecmp (p, ".gz") == 0
		   || g_ascii_strcasecmp (p, ".zip") == 0);
  use_bzip2 = p && g_ascii_strcasecmp (p, ".bz2") == 0;

  if (use_gzip || use_bzip2)
    {
/*************************************************************************/
/**     first, see if gzip can determine the uncompressed file size     **/
/**     by running 'gzip -ldq <filename>'                               **/
/*************************************************************************/

      gint field1 = 0, field2 = 0;
      gchar *pipe_cmd;

      if (use_gzip)
	{
	  pipe_cmd = g_strdup_printf (GZIP_EXE " -ldq %s", file_name);
	  fp = popen (pipe_cmd, "r");
	  g_free (pipe_cmd);

	  if (!fp)
	    return 0;
	  if (2 != fscanf (fp, "%d %d", &field1, &field2))
	    {
	      /* gzip reported something other than the size
	       * - assume some sort of error */
	      fclose (fp);
	      return 0;
	    }
	  fclose (fp);
	}

/*************************************************************************/
/**     reopen pipe to read the compressed file                         **/
/*************************************************************************/

      pipe_cmd = g_strdup_printf ("%s -cdq %s",
				  use_gzip ? GZIP_EXE : BZIP2_EXE, file_name);
      fp = popen (pipe_cmd, "r");

      if (!fp)
	return 0;

      if (field2 > 0)
	alot = (size_t) field2;	/* got file size! */
      else
	{

/*************************************************************************/
/**     gzip couldn't get file size                                     **/
/**     determine length by reading & discarding - URGH!                **/
/*************************************************************************/

	  alot = 0;
	  while (!feof (fp))
	    {
	      getc (fp);
	      alot++;
	    }
	  if (alot)
	    alot--;
	  pclose (fp);
	  fp = popen (pipe_cmd, "r");
	  if (!fp)
	    return 0;
	}
      g_free (pipe_cmd);
    }
  else
    {

/*************************************************************************/
/**     not a compressed file,                                          **/
/**     determine length with fseek                                     **/
/*************************************************************************/

      fp = fopen (file_name, "r");
      if (!fp)
	return 0;
      fseek (fp, 0, SEEK_END);
      alot = ftell (fp);
      rewind (fp);
    }

  if (alot == 0)
    return 0;

/*************************************************************************/
/**     perform block reads                                             **/
/*************************************************************************/

  buf = g_malloc (alot);

  nof_blocks = alot / BLOCK_SIZE;
  if (BLOCK_SIZE * nof_blocks < alot)
    nof_blocks++;

  inv_nof_blocks = 1.0 / nof_blocks;

  n = 0;
  for (block_count = 0; block_count < nof_blocks; block_count++)
    {
      n += fread (buf + n * BLOCK_SIZE, BLOCK_SIZE, 1, fp);
      if (block_count % 4 == 0 && pc)
	{
	  pc (pcdata, (inv_nof_blocks * block_count));
	}
    }

#ifndef	NDEBUG
  printf ("%u full blocks / %u bytes read\n", n, alot);
#endif

  if (use_gzip)
    pclose (fp);
  else
    fclose (fp);

  *db_size = alot;
  if (pc)
    pc (pcdata, 1.0);
  return buf;
}

gint
create_hash (HASH_TABLE_T * ht, gchar * words, size_t db_size,
	     void (pc) (void *, gdouble), void *pcdata)
{
  gint n;
  gint i, j;
  gint in_word, buf_offset;

  gint word_length_histogram[MAX_WORD_LENGTH];
  gint hash2;
  gdouble inv_nof_words;
  gdouble inv_db_size;

/*************************************************************************/
/**     count words                                                     **/
/*************************************************************************/

  ht->nof_words = 0;
  ht->dictionary_start = words;
  in_word = g_ascii_isprint (words[0]);
  inv_db_size = 1.0 / db_size;
  for (i = 0; i < db_size; i++)
    {

      if (words[i] == (gchar) 145)
	words[i] = '\'';
      if (g_ascii_isprint (words[i]))
	in_word = 1;
      if (!g_ascii_isprint (words[i]) && in_word)
	{
	  ht->nof_words++;
	  in_word = 0;
	}
      if (i % (128 * 1024) == 0 && pc)
	{
	  pc (pcdata, (inv_db_size * i * DB_WORD_COUNT_TIME));
	}
    }

#ifndef	NDEBUG
  printf ("%d words indexed\n", ht->nof_words);
#endif

  ht->words = g_malloc (ht->nof_words * sizeof (gchar *));
  inv_nof_words = 1.0 / ht->nof_words;

  if (!ht->words)
    {
      fprintf (stderr, __FILE__ ":%d: out of memory\n", __LINE__);
      return -1;
    }

/*************************************************************************/
/**     delimit strings / build histogram of word lengths               **/
/*************************************************************************/

  buf_offset = 0;
  for (i = 0; i < MAX_WORD_LENGTH; i++)
    word_length_histogram[i] = 0;
  for (i = 0; i < ht->nof_words; i++)
    {
      gint word_length = 0;

      ht->words[i] = words + buf_offset;
      for (j = buf_offset; g_ascii_isprint (words[j]); j++)
	{
	  if (g_ascii_isalnum (words[j]))
	    word_length++;
	}
      words[j] = 0;

      word_length_histogram[word_length]++;

      if (i < ht->nof_words - 1)
	for (++j; !g_ascii_isprint (words[j]); j++);
      buf_offset = j;
      if (i % (1024 * 16) == 0 && pc)
	{
	  pc (pcdata,
	      (i * inv_nof_words * DB_HISTOGRAM_TIME + DB_WORD_COUNT_TIME));
	}
    }

  ht->max_observed_word_length = 0;
  for (i = 0; i < MAX_WORD_LENGTH; i++)
    {
      if (word_length_histogram[i])
	{
	  if (i > ht->max_observed_word_length)
	    {
	      ht->max_observed_word_length = i;
	      if (i >= MAX_WORD_LENGTH)
		{
		  return -1;
		}
	    }
	}
    }

/*************************************************************************/
/**     setup hash table                                                **/
/*************************************************************************/

  n = ht->max_observed_word_length + 1;

  ht->hash2_table_key = g_malloc (n * sizeof (gint *));
  ht->hash2_table_string = g_malloc (n * sizeof (gchar **));
  ht->hash2_table_length = g_malloc (n * sizeof (gint));

  for (i = 0; i <= ht->max_observed_word_length; i++)
    {

      j = word_length_histogram[i];

/*
   printf("reserving %d hash entries for words of length %d\n",
   j,i);
 */

      ht->hash2_table_key[i] = g_malloc (j * sizeof (gint));
      ht->hash2_table_string[i] = g_malloc (j * sizeof (gchar *));
      ht->hash2_table_length[i] = 0;
    }

#ifndef	NDEBUG
  puts ("creating hash tables");
#endif

  for (i = 0; i < ht->nof_words; i++)
    {

      if (i % (1024 * 16) == 0 && pc)
	{
	  pc (pcdata, (i * inv_nof_words * DB_HASH_TIME +
		       DB_HISTOGRAM_TIME + DB_WORD_COUNT_TIME));
	}

      n = rm_str_real_len (ht->words[i]);
      assert (n <= ht->max_observed_word_length);

      make_hash2 (ht->words[i], &hash2);

      if (hash2 == 0)		/* fatal error */
	{
	  free_hash_table (ht);
	  return -1;
	}

      j = ht->hash2_table_length[n]++;

#ifndef	NDEBUG
      if (j >= word_length_histogram[n])
	{
	  printf ("too many words of length %d (j = %d)\n", n, j);
	}
#endif

      ht->hash2_table_key[n][j] = hash2;
      ht->hash2_table_string[n][j] = ht->words[i];
    }

#ifndef	NDEBUG
  puts ("hash tables ready");
#endif
  ht->valid = 1;
  if (pc)
    pc (pcdata, 1.0);
  return 0;
}

void
free_hash_table (HASH_TABLE_T * ht)
{
  gint i;

  for (i = 0; i <= ht->max_observed_word_length; i++)
    {
      free (ht->hash2_table_key[i]);
      free (ht->hash2_table_string[i]);
    }
  free (ht->hash2_table_length);
  free (ht->hash2_table_key);
  free (ht->hash2_table_string);
  free (ht->words);
  ht->valid = 0;
}