#include <stdlib.h>

#include "abstract.h"
#include "bit_code.h"
#include "bit_out.h"
#include "phrase.h"
#include "main.h"

phrase *alternating_phrases[2];
int phrase_toggler;
char *type_string[]=
{
"INVALID",
"VALID_BYTE"          ,
"VALID_PHRASE"        ,
"VALID_RLE_BYTE"      ,
"VALID_RLE_PHRASE"    ,
"VALID_ENCODED_BYTE"  ,
"VALID_ENCODED_PHRASE",
"VALID_ENCODED_RLE_BYTE",
"VALID_ENCODED_RLE_PHRASE"
};

abstract_buffer *abstract_clone(abstract_buffer *abuffer)
{
 abstract_buffer *buffer = (abstract_buffer *) malloc(sizeof(abstract_buffer));
 buffer->len = abuffer->len;
 buffer->data = (abstract_data *) malloc(sizeof(abstract_data) * abuffer->len);
 buffer->phrase_count = abuffer->phrase_count;
 buffer->kind = abuffer->kind;

 abstract_data *data = buffer->data;
 int i;
 for (i=0;i<abuffer->len;i++)
 {
  data[i].type = abuffer->data[i].type;
  data[i].ENCODED_code = abuffer->data[i].ENCODED_code;
  data[i].bit_length_ENCODED = abuffer->data[i].bit_length_ENCODED;
  data[i].RLE_code = abuffer->data[i].RLE_code;
  data[i].bit_length_RLE = abuffer->data[i].bit_length_RLE;
  data[i].count = abuffer->data[i].count;
  data[i].original_data = abuffer->data[i].original_data;
  data[i].phrase_used = abuffer->data[i].phrase_used;
  data[i].length_in_abstract_data_elements_of_single_element = abuffer->data[i].length_in_abstract_data_elements_of_single_element;
 }

 int *bytes_used_array;
 int *map;
 _code_ *code;
 phrase *phrases;
 code = (_code_ *) malloc (sizeof(_code_) * (256+PHRASES_MAX));
 bytes_used_array = (int *) malloc (sizeof(int) * (256+PHRASES_MAX));
 map = (int *) malloc (sizeof(int) * (256+PHRASES_MAX));
 phrases = (phrase *) malloc (sizeof(phrase) * (PHRASES_MAX));

 for (i=0;i<PHRASES_MAX;i++)
 {
  if (abuffer->phrases[i].len != 0)
  {
   int j;
   for (j=0;j<abuffer->phrases[i].len; j++)
   {
    phrases[i].phrasemem[j] = abuffer->phrases[i].phrasemem[j];
   }
  }
  phrases[i].len=abuffer->phrases[i].len;
  phrases[i].count=abuffer->phrases[i].count;
 }

 for (i=0;i<256+PHRASES_MAX;i++)
 {
  bytes_used_array[i] = abuffer->codes_used_array[i];
  map[i]=abuffer->map[i];
  code[i].bit_count = abuffer->code[i].bit_count;
  code[i].code = abuffer->code[i].code;
 }

 buffer->phrases = phrases;
 buffer->codes_used_array = bytes_used_array;
 buffer->map = map;
 buffer->code = code;
 buffer->different_codes_used = abuffer->different_codes_used;

 return buffer;
}

abstract_buffer *build_abstract(unsigned char *data_buf, int len)
{
 abstract_data *data;
 int i = 0;

 abstract_buffer *buffer = (abstract_buffer *) malloc(sizeof(abstract_buffer));
 buffer->kind = VALID;
 buffer->len = len;
 buffer->data = (abstract_data *) malloc(sizeof(abstract_data) * len);
 buffer->phrase_count = 0;
 data = buffer->data;

 for (i=0;i<len;i++)
 {
  data[i].type = VALID_BYTE;
  data[i].ENCODED_code = 0;
  data[i].bit_length_ENCODED = 0;
  data[i].RLE_code = 0;
  data[i].bit_length_RLE = 0;
  data[i].count = 1;
  data[i].original_data = data_buf[i];
  data[i].phrase_used = NULL;
  data[i].length_in_abstract_data_elements_of_single_element = 1;
 }

 int *bytes_used_array;
 int *map;
 _code_ *code;
 phrase *phrases;
 code = (_code_ *) malloc (sizeof(_code_) * (256+PHRASES_MAX));
 bytes_used_array = (int *) malloc (sizeof(int) * (256+PHRASES_MAX));
 map = (int *) malloc (sizeof(int) * (256+PHRASES_MAX));
 phrases = (phrase *) malloc (sizeof(phrase) * (PHRASES_MAX));
 buffer->phrases = phrases;

 for (i=0;i<PHRASES_MAX;i++)
 {
  phrases[i].len=0;
  phrases[i].count=0;
 }

 for (i=0;i<256+PHRASES_MAX;i++)
  bytes_used_array[i]=0;
 i=0;
 while( i != len )
 {
  bytes_used_array[data[i].original_data]++;
  i++;
 }

 int bytes_used_array_count=0; // count of different 'bytes' (+phrases) used in this abstract
 for (i=0;i<256+PHRASES_MAX;i++)
 {
  if (bytes_used_array[i] != 0)
  {
    bytes_used_array_count++;
  }
 }
// count bytes end

// fill map start
  int remember = -1;
  int map_count = 0;
  for (i=0;i<256+PHRASES_MAX;i++)
   map[i]=-1;

  while (map_count < 256 + PHRASES_MAX)
  {
    int most = -1;
    for (i=0;i<256+PHRASES_MAX ;i++)
    {
      if (bytes_used_array[i] >= most)
      {
        most = bytes_used_array[i];
        remember = i;
      }
    }
    map[map_count] = remember;
    bytes_used_array[remember] = -2;
    map_count++;
  }
// fill map end

// correct count start
 for (i=0;i<256+PHRASES_MAX;i++)
  bytes_used_array[i]=0;
 i=0;
 while( i != len )
 {
  bytes_used_array[data[i].original_data]++;
  i++;
 }
 buffer->codes_used_array = bytes_used_array;
 buffer->map = map;
 buffer->code = code;
 buffer->different_codes_used = bytes_used_array_count;

 return buffer;
}

void delete_abstract(abstract_buffer *buffer)
{
 free(buffer->phrases);
 free(buffer->data);
 free(buffer->code);
 free(buffer->codes_used_array);
 free(buffer->map);
 free(buffer);
}

void abstract_Huffman(abstract_buffer *buffer)
{
// count bytes start
 abstract_data *data = buffer->data;
 int len = buffer->len;
 int i = 0;
 buffer->kind = ENCODED;

 // result in code[][]
 hstart(0, buffer->different_codes_used, buffer->codes_used_array, buffer->map, buffer->code);

 i=0;
 while( i < len )
 {
  if (data[i].type == VALID_BYTE)
  {
   data[i].type = VALID_ENCODED_BYTE;

if (buffer->code[data[i].original_data].code>255)
{
 printf("encode error, code = %i (from %i)\n", buffer->code[data[i].original_data].code, data[i].original_data);
}
   data[i].ENCODED_code = buffer->code[data[i].original_data].code;
   data[i].bit_length_ENCODED = buffer->code[data[i].original_data].bit_count;
  }
  else if (data[i].type == VALID_PHRASE)
  {
   data[i].type = VALID_ENCODED_PHRASE;
if (buffer->code[data[i].phrase_used+256].code>255)
{
 printf("encode error!");
 exit(1);
}
   data[i].ENCODED_code = buffer->code[data[i].phrase_used+256].code;
   data[i].bit_length_ENCODED = buffer->code[data[i].phrase_used+256].bit_count;
  }
  i+=data[i].length_in_abstract_data_elements_of_single_element;
 }
}

void abstract_Shannon(abstract_buffer *buffer)
{
// count bytes start
 abstract_data *data = buffer->data;
 int len = buffer->len;
 int i;
 buffer->kind = ENCODED;

 // result in code[][]
 shannon(0, buffer->different_codes_used, 0, 0, len, buffer->codes_used_array, buffer->map, buffer->code);

 i=0;
 while( i < len )
 {
  if (data[i].type == VALID_BYTE)
  {
   data[i].type = VALID_ENCODED_BYTE;
if (buffer->code[data[i].original_data].code>255)
{
 printf("encode error!");
 exit(1);
}
   data[i].ENCODED_code = buffer->code[data[i].original_data].code;
   data[i].bit_length_ENCODED = buffer->code[data[i].original_data].bit_count;
  }
  else if (data[i].type == VALID_PHRASE)
  {
   data[i].type = VALID_ENCODED_PHRASE;
if (buffer->code[data[i].phrase_used+256].code>255)
{
 printf("encode error!");
 exit(1);
}
   data[i].ENCODED_code = buffer->code[data[i].phrase_used+256].code;
   data[i].bit_length_ENCODED = buffer->code[data[i].phrase_used+256].bit_count;
  }
  i+=data[i].length_in_abstract_data_elements_of_single_element;
 }
}

void abstract_RLE(abstract_buffer *buffer)
{
  if (buffer->kind == ENCODED)
   buffer->kind = RLE_ENCODED;
  else
   buffer->kind = RLE;

  abstract_data *data = buffer->data;
  int len = buffer->len;
  int pos = 0;
  int current_out = 0;
  int current_out_bit = 0;
  int current_code = 0;
  int current_bits = 0;
  while( pos < len )
  {
   int counter = 1;
// fixme test for valid data
// fixme for non ENCODED
   if (  (data[pos].type == VALID_ENCODED_BYTE)
      || (data[pos].type == VALID_ENCODED_PHRASE)
      )
   {
    current_code = data[pos].ENCODED_code;
    current_bits = data[pos].bit_length_ENCODED;
   }
   else if (data[pos].type == VALID_BYTE)
   {
    current_code = data[pos].original_data;
    current_bits = 8;
   }
   else if (data[pos].type == VALID_PHRASE)
   {
    current_code = data[pos].phrase_used;
    current_bits = 9;
   }

   while (pos+data[pos].length_in_abstract_data_elements_of_single_element < len)
   {
// fixme for non ENCODED
     if (
          (((data[pos].type == VALID_ENCODED_BYTE) && (data[pos+data[pos].length_in_abstract_data_elements_of_single_element].type == VALID_ENCODED_BYTE)) &&
           (data[pos+data[pos].length_in_abstract_data_elements_of_single_element].ENCODED_code == data[pos].ENCODED_code) &&
           (data[pos+data[pos].length_in_abstract_data_elements_of_single_element].bit_length_ENCODED == data[pos].bit_length_ENCODED))
         ||
          (((data[pos].type == VALID_ENCODED_PHRASE) && (data[pos+data[pos].length_in_abstract_data_elements_of_single_element].type == VALID_ENCODED_PHRASE)) &&
           (data[pos+data[pos].length_in_abstract_data_elements_of_single_element].ENCODED_code == data[pos].ENCODED_code) &&
           (data[pos+data[pos].length_in_abstract_data_elements_of_single_element].bit_length_ENCODED == data[pos].bit_length_ENCODED))
         ||
          (((data[pos].type == VALID_BYTE) && (data[pos+data[pos].length_in_abstract_data_elements_of_single_element].type == VALID_BYTE)) &&
           (data[pos+data[pos].length_in_abstract_data_elements_of_single_element].original_data == data[pos].original_data))
         ||
          (((data[pos].type == VALID_PHRASE) && (data[pos+data[pos].length_in_abstract_data_elements_of_single_element].type == VALID_PHRASE)) &&
           (data[pos+data[pos].length_in_abstract_data_elements_of_single_element].phrase_used == data[pos].phrase_used))
        )
     {
      data[pos].type = INVALID;
      // the last occurance is valid
      pos += data[pos].length_in_abstract_data_elements_of_single_element;
      counter++;
      continue;
     }
     break;
    }
    if (counter == 1)
    {
// fixme for non ENCODED
     if ((data[pos].type == VALID_ENCODED_BYTE) || (data[pos].type == VALID_ENCODED_PHRASE))
     {
      data[pos].RLE_code = 0;
      data[pos].bit_length_RLE = 1;
      data[pos].count = 1;
      if (data[pos].type == VALID_ENCODED_BYTE)
       data[pos].type = VALID_ENCODED_RLE_BYTE;
      else if (data[pos].type == VALID_ENCODED_PHRASE)
       data[pos].type = VALID_ENCODED_RLE_PHRASE;
     }
     else
     {
      if ((data[pos].type == VALID_BYTE) || (data[pos].type == VALID_PHRASE))
      {
       data[pos].RLE_code = 0;
       data[pos].bit_length_RLE = 1;
       data[pos].count = 1;
       if (data[pos].type == VALID_BYTE)
        data[pos].type = VALID_RLE_BYTE;
       else if (data[pos].type == VALID_PHRASE)
        data[pos].type = VALID_RLE_PHRASE;
      }
     }
    }
    else
    {
     int bits_for_counter = get_bits_for_counter(counter);

     if ((data[pos].type == VALID_ENCODED_BYTE) || (data[pos].type == VALID_ENCODED_PHRASE))
     {
      data[pos].RLE_code = get_RLE_code(counter);
      data[pos].bit_length_RLE = bits_for_counter - 1 + bits_for_counter;
      data[pos].count = counter;
      if (data[pos].type == VALID_ENCODED_BYTE)
       data[pos].type = VALID_ENCODED_RLE_BYTE;
      else if (data[pos].type == VALID_ENCODED_PHRASE)
       data[pos].type = VALID_ENCODED_RLE_PHRASE;
     }
     else
     {
      if ((data[pos].type == VALID_BYTE) || (data[pos].type == VALID_PHRASE))
      {
       data[pos].RLE_code = get_RLE_code(counter);
       data[pos].bit_length_RLE = bits_for_counter - 1 + bits_for_counter;
       data[pos].count = counter;
       if (data[pos].type == VALID_BYTE)
        data[pos].type = VALID_RLE_BYTE;
       else if (data[pos].type == VALID_PHRASE)
        data[pos].type = VALID_RLE_PHRASE;
      }
     }
    }
    pos+=data[pos].length_in_abstract_data_elements_of_single_element;
  }
}


int get_bits_used_from_abstract(abstract_buffer *buffer)
{
 abstract_data *data = buffer->data;
 int len = buffer->len;
 int count = 0;
 int i=0;
 while( i < len )
 {
  if (data[i].type == VALID_BYTE)
  {
   count += 8;
  }

  if (data[i].type == VALID_PHRASE)
  {
   count += 9; // non encoded phrase 'more' than one byte
  }

  if (data[i].type == VALID_RLE_BYTE)
  {
   count += 8;
   count += data[i].bit_length_RLE;
  }

  if (data[i].type == VALID_RLE_PHRASE)
  {
   count += 9;
   count += data[i].bit_length_RLE;
  }

  if (   (data[i].type == VALID_ENCODED_BYTE)
      || (data[i].type == VALID_ENCODED_PHRASE)
     )
  {
   count += data[i].bit_length_ENCODED;
  }

  if (   (data[i].type == VALID_ENCODED_RLE_BYTE)
      || (data[i].type == VALID_ENCODED_RLE_PHRASE)
     )
  {
   count += data[i].bit_length_RLE;
   count += data[i].bit_length_ENCODED;
  }
  i+=data[i].length_in_abstract_data_elements_of_single_element;
 }
 return count;
}

int get_bits_used_from_abstract_complete(abstract_buffer *buffer)
{
 int used_bits = get_bits_used_from_abstract(buffer);
 int i;
// used_bits += 8; // buffer->different_codes_used
 for(i=0;i<buffer->different_codes_used;i++)
 {
  // not exact for non optimal phase
  // since count 0 phrases phrases can occur!
  used_bits += 8+8+8; // buffer->code[buffer->map[i]].bit_count, buffer->code[buffer->map[i]].code,buffer->map[i]
 }

 i=0;
 while (buffer->phrases[i].len != 0)
 {
  // not exact for non optimal phase
  // since count 0 phrases phrases can occur!
  used_bits += 8; // len of phrase
  int j;
  for (j=0; j<buffer->phrases[i].len; j++)
  {
   // for now only unpacked data == bytes
   unsigned char outer = (unsigned char) (buffer->phrases[i].phrasemem[j]);
   used_bits += 8; // a phrase code
  }
  i++;
 }
 return used_bits;
}

struct sorted_codes
{
 unsigned char bits;
 unsigned char code;
 unsigned char value;
 int map;
};

int sorted_compare( const void *op1, const void *op2 )
{
 const sorted_codes *p1 = (const sorted_codes *) op1;
 const sorted_codes *p2 = (const sorted_codes *) op2;
 if (p1->bits<p2->bits) return -1;
 if (p1->bits>p2->bits) return 1;
 if (p1->code<p2->code) return -1;
 if (p1->code>p2->code) return 1;
 return 0;
}

sorted_codes *abstract_sort_code_with_bits(abstract_buffer *buffer)
{
 int i;
 sorted_codes *sorted = (sorted_codes *)malloc(sizeof(sorted_codes) * buffer->different_codes_used);
 for(i=0;i<buffer->different_codes_used;i++)
 {
  sorted[i].bits = (unsigned char) (buffer->code[buffer->map[i]].bit_count);
  sorted[i].code = (unsigned char)(buffer->code[buffer->map[i]].code);
  sorted[i].value = (unsigned char)((buffer->map[i])&255);
  sorted[i].map = buffer->map[i];
 }
 qsort( sorted, buffer->different_codes_used, sizeof(sorted_codes), sorted_compare );
 return sorted;
}

void abstract_out(abstract_buffer *buffer)
{
 abstract_data *data = buffer->data;
 int len = buffer->len;
 int i = 0;
 int j = 0;
 FILE* outFile = get_dbOutFile();
 FILE* outFile2 = get_binOutFile();
 if ((buffer->kind == ENCODED) || (buffer->kind == RLE_ENCODED))
 {
  if (outFile != NULL)
  {
   fprintf(outFile,"; translation data \n");
   if (COUT_PUT==1)
   {
    fprintf(outFile,"; .byte 0H%02X; bytes follow \n", buffer->different_codes_used);
   }
   else
   {
    fprintf(outFile,"; DB $%02X; bytes follow \n", buffer->different_codes_used);
   }
   fprintf(outFile,"; bits used, code, real 'byte' \n");
   fprintf(outFile,"%s_reg_%i: \n", file_name, current_working_register);
  }
  if (outFile2 != NULL)
  {
   fwrite(((unsigned char *)(&buffer->different_codes_used)),1,1,outFile2);
  }
  sorted_codes *sorted = abstract_sort_code_with_bits(buffer);
  for(i=0;i<buffer->different_codes_used;i++)
  {
   unsigned char bit_count = (unsigned char) (sorted[i].bits);
   unsigned char code = (unsigned char)(sorted[i].value);
   int count;
   if (sorted[i].map>=256)
   {
    // bit one of bit count is pointer to whether it is a phrase or a
    // byte
    bit_count+=128;
   }
   if (bit_count>128)
    count = buffer->codes_used_array[256+code];
   else
    count = buffer->codes_used_array[code];
   if (count > 0)
   {
    if (outFile != NULL)
    {
     if (COUT_PUT==1)
     {
      fprintf(outFile," .byte 0H%02X, 0H%02X, 0H%02X ;%i \n", bit_count, sorted[i].code, code , count);
     }
     else
     {
      fprintf(outFile," DB $%02X, $%02X, $%02X ;%i \n", bit_count, sorted[i].code, code , count);
     }
    }
    if (outFile2 != NULL)
    {
     fwrite(((unsigned char *)(&bit_count)),1,1,outFile2);
     fwrite(((unsigned char *)(&sorted[i].code)),1,1,outFile2);
     fwrite(((unsigned char *)(&code)),1,1,outFile2);
    }
   }
  }
  free(sorted);
  if (USE_PHRASE)
  {
   if (outFile != NULL)
   {
    fprintf(outFile,"; phrases follow \n");
    fprintf(outFile,"%s_pd_%i: \n", file_name, current_working_register);
   }
   i=0;
   while (buffer->phrases[i].len != 0)
   {
    if (buffer->codes_used_array[i+256]>0)
    {
     if (outFile != NULL)
     {
      if (COUT_PUT==1)
      {
       fprintf(outFile," .byte 0H%02X", buffer->phrases[i].len);
      }
      else
      {
       fprintf(outFile," DB $%02X", buffer->phrases[i].len);
      }
     }
     if (outFile2 != NULL)
     {
      fwrite(((unsigned char *)(&buffer->phrases[i].len)),1,1,outFile2);
     }
     int j;
     for (j=0; j<buffer->phrases[i].len; j++)
     {
      // for now only unpacked data == bytes
      unsigned char outer = (unsigned char) (buffer->phrases[i].phrasemem[j]);
      if (outFile != NULL)
      {
       if (COUT_PUT==1)
       {
        fprintf(outFile,", 0H%02X", outer);
       }
       else
       {
        fprintf(outFile,", $%02X", outer);
       }
      }
      if (outFile2 != NULL)
      {
       fwrite(((unsigned char *)(&outer)),1,1,outFile2);
      }
     }
     if (j!=0)
     {
      if (outFile != NULL)
      {
       fprintf(outFile,"; %i \n", buffer->codes_used_array[i+256]);
      }
     }
    }
    i++;
   }
  }
 }
 if (outFile != NULL)
 {
  fprintf(outFile,"; data follows \n");
  fprintf(outFile,"%s_reg_%i_data: \n", file_name, current_working_register);
 }

 i = 0;
 while( i < len )
 {
// invalid are not used!
  if (data[i].type == VALID_BYTE)
  {
   byte_out((unsigned char)data[i].original_data);
  }

  if (data[i].type == VALID_PHRASE)
  {
   int j;
   for (j=0;j<buffer->phrases[data[i].phrase_used].len; j++)
   {
    byte_out((unsigned char)buffer->phrases[data[i].phrase_used].phrasemem[j]);
   }
  }

  if (data[i].type == VALID_RLE_BYTE)
  {
   bits_out_bit_code(data[i].bit_length_RLE, data[i].RLE_code);
   byte_out((unsigned char)data[i].original_data);
  }

  if (data[i].type == VALID_RLE_PHRASE)
  {
   bits_out_bit_code(data[i].bit_length_RLE, data[i].RLE_code);
   int j;
   for (j=0;j<buffer->phrases[data[i].phrase_used].len; j++)
   {
    byte_out((unsigned char)buffer->phrases[data[i].phrase_used].phrasemem[j]);
   }
  }

  if (data[i].type == VALID_ENCODED_BYTE)
  {
   bits_out_bit_code(data[i].bit_length_ENCODED, data[i].ENCODED_code);
  }

  if (data[i].type == VALID_ENCODED_PHRASE)
  {
   bits_out_bit_code(data[i].bit_length_ENCODED, data[i].ENCODED_code);
  }

  if (data[i].type == VALID_ENCODED_RLE_BYTE)
  {
   bits_out_bit_code(data[i].bit_length_RLE, data[i].RLE_code);
   bits_out_bit_code(data[i].bit_length_ENCODED, data[i].ENCODED_code);
  }

  if (data[i].type == VALID_ENCODED_RLE_PHRASE)
  {
   bits_out_bit_code(data[i].bit_length_RLE, data[i].RLE_code);
   bits_out_bit_code(data[i].bit_length_ENCODED, data[i].ENCODED_code);
  }
  i+=data[i].length_in_abstract_data_elements_of_single_element;
 }
 bit_flush();
}

int abstract_phrase_already_seen(phrase *phrases, int *phrase_codes)
{
  int current_phrase = 0;
  while (phrases[current_phrase].len != 0)
  {
   int i;
   for (i=0;i<phrases[current_phrase].len;i++)
   {
    if (phrases[current_phrase].phrasemem[i] != phrase_codes[i])
    {
     break;
    }
   }
   if (i == phrases[current_phrase].len)
   {
    if (PHRASE_OPTIMIZER != 2)
    {
     phrases[current_phrase].count++;
    }
    return 1;
   }
   current_phrase++;
  }
  return 0;
}

int abstract_phrase_valid(phrase *phrases, int *phrase_codes, int phrase_len)
{
 int i;
 int first = phrase_codes[0];
 for (i=0;i<phrase_len;i++)
 {
  if (phrase_codes[i] != first)
  {
   break;
  }
 }
 if (i == phrase_len)
 {
  return 0;
 }
 return !abstract_phrase_already_seen(phrases, phrase_codes);
}

int abstract_get_phrase_count(abstract_buffer *abuffer, int *phrase_codes, int phrase_len, int start)
{
 int counter = 0;
 abstract_data *data = abuffer->data;
 int array_counter = start;
 while (array_counter + phrase_len < abuffer->len)
 {
  int j;
  for (j=0; j<phrase_len;j++)
  {
   if (data[array_counter].type != VALID_BYTE)
    break;
   if (phrase_codes[j] == data[array_counter].original_data)
   {
    if (array_counter < abuffer->len)
    {
      if (j+1<phrase_len)
       array_counter++;
    }
    else
    {
     return counter;
    }
   }
   else
   {
    break;
   }
  }
  if (j==phrase_len)
  {
   counter++;
  }
  array_counter++;
 }
 return counter;
}

// time killer!!!
phrase abstract_get_best_phrase_used_more_than_once(abstract_buffer *abuffer, int phrase_len)
{
  int phrase_codes[_MAX_PHRASE_LEN_];
  int phrase_count=abuffer->len-(phrase_len-1);
  if (alternating_phrases[phrase_toggler] == NULL)
   alternating_phrases[phrase_toggler] = (phrase *) malloc( sizeof(phrase) * (phrase_count+1) );
  abstract_data *data = abuffer->data;
  alternating_phrases[phrase_toggler][0].len = 0;
  alternating_phrases[phrase_toggler][0].count = 0;

  int current_phrase = 0;
  int array_counter = 0;

  int max_count = 1;
  int best_index = -1;
  phrase best;
  best.len = 0;
  int other_toggle = (phrase_toggler+1)%2;
  while (array_counter+phrase_len<abuffer->len)
  {
   int i;
   int valid =1 ;

   if (alternating_phrases[other_toggle] != NULL)
   {
    valid = 0;
    i=0;
    while (alternating_phrases[other_toggle][i].len != 0)
    {
     if (alternating_phrases[other_toggle][i].start == array_counter)
     {
      valid = 1;
      break;
     }
     i++;
    }
   }

   if (valid)
   {
    for (i=0; i<phrase_len;i++)
    {
     if (data[array_counter+i].type != VALID_BYTE)
     {
      valid = 0;
      break;
     }
     phrase_codes[i] = data[array_counter+i].original_data;
    }
    if (valid == 0)
    {
     array_counter++;
     continue;
    }
    // in abstract_phrase_already_seen(), which is called in
    // abstract_phrase_valid the count for doubles is set!
    // above is true for search optimization = 0!
    if (abstract_phrase_valid(alternating_phrases[phrase_toggler], phrase_codes, phrase_len))
    {
     if (PHRASE_OPTIMIZER == 0)
     {
      alternating_phrases[phrase_toggler][current_phrase].len = phrase_len;
      alternating_phrases[phrase_toggler][current_phrase].count = 1;
      alternating_phrases[phrase_toggler][current_phrase].start = array_counter;
      for (i=0; i<phrase_len;i++)
      {
        alternating_phrases[phrase_toggler][current_phrase].phrasemem[i] = phrase_codes[i];
      }
      current_phrase++;

      alternating_phrases[phrase_toggler][current_phrase].len = 0;
      alternating_phrases[phrase_toggler][current_phrase].count = 0;
     }
     else
     {
      // takes much time, but is difficult to correctly count in another manor
      // since overlapping (same) phrases can occur, which garbles count
      int count = abstract_get_phrase_count(abuffer, phrase_codes, phrase_len, array_counter+1);
      if (count > 1)
      {
       alternating_phrases[phrase_toggler][current_phrase].len = phrase_len;
       alternating_phrases[phrase_toggler][current_phrase].count = count;
       alternating_phrases[phrase_toggler][current_phrase].start = array_counter;
       for (i=0; i<phrase_len;i++)
       {
         alternating_phrases[phrase_toggler][current_phrase].phrasemem[i] = phrase_codes[i];
       }
       if (count > max_count)
       {
        max_count = count;
        best_index = current_phrase;
       }
       current_phrase++;

       alternating_phrases[phrase_toggler][current_phrase].len = 0;
       alternating_phrases[phrase_toggler][current_phrase].count = 0;
      }
     }
    }
   }
   array_counter++;
  }

  if (PHRASE_OPTIMIZER == 0)
  {
   for (int i=0; i < current_phrase; i++)
   {
    if (alternating_phrases[phrase_toggler][i].count > max_count)
    {
     max_count = alternating_phrases[phrase_toggler][i].count;
     best_index = i;
    }
   }
  }

  if (best_index != -1)
  {
   best.len = alternating_phrases[phrase_toggler][best_index].len;
   best.count = alternating_phrases[phrase_toggler][best_index].count;
   for (int i=0; i<phrase_len;i++)
   {
     best.phrasemem[i] = alternating_phrases[phrase_toggler][best_index].phrasemem[i];
   }
  }
  if (PHRASE_OPTIMIZER != 2)
  {
   phrase_toggler=(phrase_toggler+1)%2;
  }
  return best;
}


void abstract_reset_buffer(abstract_buffer *ubuffer, abstract_buffer *obuffer)
{
 ubuffer->len = obuffer->len;
 ubuffer->phrase_count = obuffer->phrase_count;
 abstract_data *data = ubuffer->data;
 int i;
 for (i=0;i<obuffer->len;i++)
 {
  data[i].type = obuffer->data[i].type;
  data[i].ENCODED_code = obuffer->data[i].ENCODED_code;
  data[i].bit_length_ENCODED = obuffer->data[i].bit_length_ENCODED;
  data[i].RLE_code = obuffer->data[i].RLE_code;
  data[i].bit_length_RLE = obuffer->data[i].bit_length_RLE;
  data[i].count = obuffer->data[i].count;
  data[i].original_data = obuffer->data[i].original_data;
  data[i].phrase_used = obuffer->data[i].phrase_used;
  data[i].length_in_abstract_data_elements_of_single_element = obuffer->data[i].length_in_abstract_data_elements_of_single_element;
 }
 for (i=0;i<PHRASES_MAX;i++)
 {
  if (obuffer->phrases[i].len != 0)
  {
   int j;
   for (j=0;j<obuffer->phrases[i].len; j++)
   {
    ubuffer->phrases[i].phrasemem[j] = obuffer->phrases[i].phrasemem[j];
   }
  }
  ubuffer->phrases[i].len=obuffer->phrases[i].len;
  ubuffer->phrases[i].count=obuffer->phrases[i].count;
 }

 for (i=0;i<256+PHRASES_MAX;i++)
 {
  ubuffer->codes_used_array[i] = obuffer->codes_used_array[i];
  ubuffer->map[i]=obuffer->map[i];
  ubuffer->code[i].bit_count = obuffer->code[i].bit_count;
  ubuffer->code[i].code = obuffer->code[i].code;
 }
 ubuffer->different_codes_used = obuffer->different_codes_used;
}

void abstract_apply_phrase(abstract_buffer *abuffer, phrase aphrase)
{
 int phrase_count = abuffer->phrase_count;
 int i;
 if (phrase_count >= PHRASES_MAX)
 {
  return;
 }
 abuffer->phrase_count++;

 for (i=0; i<aphrase.len; i++)
 {
  abuffer->phrases[phrase_count].phrasemem[i] = aphrase.phrasemem[i];
 }
 abuffer->phrases[phrase_count].count = 0;
 abuffer->phrases[phrase_count].len = aphrase.len;
 abstract_data *data = abuffer->data;

 // now scan data for phrase
 // only for original data for now
 int data_pointer=0;
 while (data_pointer < abuffer->len)
 {
  int j = 0;
  int search_start = data_pointer;
  while ((data_pointer < abuffer->len) && (j < aphrase.len))
  {
   if (data[data_pointer].type != VALID_BYTE)
   {
    break;
   }
   if (data[data_pointer].original_data == aphrase.phrasemem[j])
   {
    j++;
    data_pointer++;
    continue;
   }
   break;
  }
  if (j == aphrase.len)
  {
//printf(" \n Phrase found at: %i \n", search_start);
   // phrase found
   int j = 0;
   data_pointer = search_start;
   while (j < aphrase.len)
   {
    if (j == 0)
    {
     data[data_pointer].type = VALID_PHRASE;
     data[data_pointer].phrase_used = phrase_count;
     data[data_pointer].length_in_abstract_data_elements_of_single_element = aphrase.len;
     abuffer->phrases[phrase_count].count++;
//printf("# Phrase used: %i, length: %i \n", data[data_pointer].phrase_used, data[data_pointer].length_in_abstract_data_elements_of_single_element);
    }
    else
    {
     data[data_pointer].type = INVALID;
    }
    j++;
    data_pointer++;
   }
   data_pointer--;
  }
  data_pointer++;
 }

 // correct map, code, codes_used_array
 for (i=0;i < aphrase.len; i++)
 {
  int current_code = aphrase.phrasemem[i];
  abuffer->codes_used_array[current_code] -= (abuffer->phrases[phrase_count].count-1);
 }
 abuffer->codes_used_array[256+phrase_count] = aphrase.len * abuffer->phrases[phrase_count].count;
 abuffer->different_codes_used++;

 int *bytes_used_array;
 bytes_used_array = (int *) malloc (sizeof(int) * (256+PHRASES_MAX));

// fill map start
 int map_count = 0;
 for (i=0;i<256+PHRASES_MAX;i++)
 {
  abuffer->map[i]=-1;
  bytes_used_array[i] = abuffer->codes_used_array[i];
 }
 int remember = -1;
 while (map_count < 256 + PHRASES_MAX)
 {
  int most = -1;
  for (i=0;i<256+PHRASES_MAX ;i++)
  {
   if (bytes_used_array[i] >= most)
   {
     most = bytes_used_array[i];
     remember = i;
   }
  }
  abuffer->map[map_count] = remember;
  bytes_used_array[remember] = -2;
  map_count++;
 }
// fill map end
 free(bytes_used_array);
}

phrase abstract_get_best_single_phrase(abstract_buffer *abuffer)
{
 phrase phrase_to_remember;
 phrase_to_remember.len = 0;
 abstract_buffer *local_buffer;
 local_buffer = abstract_clone(abuffer);
 int used_bits_without_phrase = get_bits_used_from_abstract_complete(local_buffer);
 int minimum_bits_so_far = used_bits_without_phrase;

 int phrase_len;
 printf("Testing phrase with len: ");
 alternating_phrases[0] = NULL;
 alternating_phrases[1] = NULL;
 phrase_toggler=0;
 for (phrase_len=MIN_PHRASE_LEN; phrase_len<MAX_PHRASE_LEN;phrase_len++)
 {
  printf(" %i", phrase_len);
  fflush(stdout);
  phrase best_phrase = abstract_get_best_phrase_used_more_than_once(local_buffer, phrase_len);
  if (best_phrase.len == 0)
   continue;
  abstract_apply_phrase(local_buffer, best_phrase);
  if (SHANNON)
   abstract_Shannon(local_buffer);
  if (RLE_USED)
   abstract_RLE(local_buffer);
  int used_bits = get_bits_used_from_abstract_complete(local_buffer);
  if (minimum_bits_so_far>used_bits)
  {
   minimum_bits_so_far=used_bits;
   phrase_to_remember.len = best_phrase.len;
   phrase_to_remember.count = best_phrase.count;
   int i;
   for (i=0; i< best_phrase.len; i++)
   {
    phrase_to_remember.phrasemem[i] = best_phrase.phrasemem[i];
   }
  }
  abstract_reset_buffer(local_buffer, abuffer);
 }
 delete_abstract(local_buffer);
 if (alternating_phrases[0] != NULL)
  free(alternating_phrases[0]);
 if (alternating_phrases[1] != NULL)
  free(alternating_phrases[1]);
 printf(" \n");
 return phrase_to_remember;
}

void abstract_search_insert_phrases(abstract_buffer *abuffer)
{
  abstract_buffer *local_buffer = abstract_clone(abuffer);
  if (SHANNON)
   abstract_Shannon(local_buffer);
  if (RLE_USED)
   abstract_RLE(local_buffer);
  int used_bits = get_bits_used_from_abstract_complete(local_buffer);
  printf("Non Phrase: Reg: %i, bytes used: %i  \n", current_working_register, (7+used_bits)/8);
  delete_abstract(local_buffer);

  phrase phrases[PHRASES_MAX];
  int phrases_depth;
  for (phrases_depth=0;phrases_depth<PHRASES_MAX_DEPTH;phrases_depth++)
  {
   phrase aphrase = abstract_get_best_single_phrase(abuffer);
   if (aphrase.len == 0)
   {
    printf("No phrase found! \n");
    continue;
   }
   phrases[phrases_depth].len = aphrase.len;
   phrases[phrases_depth].count = aphrase.count;
   int i;
   for (i=0;i<aphrase.len;i++)
   {
    phrases[phrases_depth].phrasemem[i] = aphrase.phrasemem[i];
   }
   abstract_apply_phrase(abuffer, aphrase);

   abstract_buffer *local_buffer = abstract_clone(abuffer);
   if (SHANNON)
    abstract_Shannon(local_buffer);
   if (RLE_USED)
    abstract_RLE(local_buffer);
   int used_bits = get_bits_used_from_abstract_complete(local_buffer);
   delete_abstract(local_buffer);
   printf("Last result of test: Reg: %i, Depth: %i, phrase_len: %i, bytes used: %i  \n", current_working_register, phrases_depth, aphrase.len, (7+used_bits)/8);
  }
}

void abstract_search_insert_phrases_optimal(abstract_buffer *abuffer)
{
  abstract_buffer *local_buffer = abstract_clone(abuffer);
  if (SHANNON)
   abstract_Shannon(local_buffer);
  if (RLE_USED)
   abstract_RLE(local_buffer);
  phrase aphrase;
  int used_bits_last;
  int used_bits = get_bits_used_from_abstract_complete(local_buffer);
  int best_used = used_bits;
  printf("Non Phrase: Reg: %i, bytes used: %i  \n", current_working_register, (7+used_bits)/8);
  int phrases_depth = 0;
  int phrase_used = 0;

  delete_abstract(local_buffer);
  do
  {
   used_bits_last = used_bits;
   aphrase = abstract_get_best_single_phrase(abuffer);
   if (aphrase.len == 0)
   {
    printf("No phrase found! \n");
    continue;
   }

   local_buffer = abstract_clone(abuffer);
   abstract_apply_phrase(local_buffer, aphrase);
   if (SHANNON)
    abstract_Shannon(local_buffer);
   if (RLE_USED)
    abstract_RLE(local_buffer);
   used_bits = get_bits_used_from_abstract_complete(local_buffer);
   delete_abstract(local_buffer);
   if (used_bits<used_bits_last)
   {
    abstract_apply_phrase(abuffer, aphrase);
    printf("Last result of test: Reg: %i, Depth: %i, phrase_len: %i, bytes used: %i  \n", current_working_register, phrases_depth, aphrase.len, (7+used_bits)/8);
    best_used = used_bits;
    phrase_used = 1;
   }
   phrases_depth++;
  }
  while ((used_bits<used_bits_last) &&(phrases_depth<PHRASES_MAX));
  if (phrase_used == 1)
   printf("Optimum reached: Reg: %i, Depth: %i, phrase_len: %i, bytes used: %i  \n", current_working_register, phrases_depth, aphrase.len, (7+best_used)/8);
  else
   printf("Optimum reached: Reg: %i, bytes used: %i (no phrase used) \n", current_working_register, (7+best_used)/8);
}

void print_abstract(abstract_buffer *abuffer)
{
 int i;
 for (i=0;i<abuffer->len;i++)
 {
  printf("%i.--- \n",i);
  printf("type      :%s \n", type_string[abuffer->data[i].type]);
  printf("count     :%i \n", abuffer->data[i].count);
  printf("odata     :%i \n", abuffer->data[i].original_data);
  printf("phrase    :%i \n", abuffer->data[i].phrase_used);
  printf("data_len  :%i \n", abuffer->data[i].length_in_abstract_data_elements_of_single_element);
 }
 printf("\n");
 printf("Other abstract data: \n");
 printf("len        :%i \n", abuffer->len);
 printf("#phrases   :%i \n", abuffer->phrase_count);
 printf("codes used :%i \n", abuffer->different_codes_used);
 printf("kind       :%i \n", abuffer->kind);
 printf("\n");
 printf("Phrases: \n");
 for (i=0;i<abuffer->phrase_count;i++)
 {
  int j;
  printf("Phrase %i: \n",i);
  for (j=0;j<abuffer->phrases[i].len; j++)
  {
   printf("%i ",abuffer->phrases[i].phrasemem[j]);
  }
  printf(" \n");
 }

}