MtasFieldsConsumer.java

package mtas.codec;

import java.io.Closeable;
import java.io.EOFException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map.Entry;
import java.util.SortedMap;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;

import mtas.analysis.token.MtasOffset;
import mtas.analysis.token.MtasPosition;
import mtas.analysis.token.MtasToken;
import mtas.analysis.token.MtasTokenString;
import mtas.codec.payload.MtasPayloadDecoder;
import mtas.codec.tree.MtasRBTree;
import mtas.codec.tree.MtasTree;
import mtas.codec.tree.MtasTreeNode;
import mtas.codec.tree.MtasTreeNodeId;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.MappedMultiFields;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.ReaderSlice;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;

/**
 * The Class MtasFieldsConsumer.
 */

/**
 * The Class MtasFieldsConsumer constructs several temporal and permanent files
 * to provide a forward index
 *
 * <ul>
 * <li><b>Temporary files</b><br>
 * <ul>
 * <li><b>Temporary file {@link #mtasTmpFieldFileName} with extension
 * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_TMP_FIELD_EXTENSION} </b><br>
 * Contains for each field a reference to the list of documents. Structure of
 * content:
 * <ul>
 * <li><b>String</b>: field</li>
 * <li><b>VLong</b>: reference to {@link #mtasDocFileName}</li>
 * <li><b>VInt</b>: number of documents</li>
 * <li><b>VLong</b>: reference to {@link #mtasTermFileName}</li>
 * <li><b>VInt</b>: number of terms</li>
 * <li><b>VLong</b>: reference to {@link #mtasPrefixFileName}</li>
 * <li><b>VInt</b>: number of prefixes</li>
 * </ul>
 * </li>
 * <li><b>Temporary file {@link #mtasTmpObjectFileName} with extension
 * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_TMP_OBJECT_EXTENSION}</b><br>
 * Contains for a specific field all objects constructed by
 * {@link createObjectAndRegisterPrefix}. For all fields, the objects are later
 * on copied to {@link #mtasObjectFileName} while statistics are collected.
 * Structure of content identical to {@link #mtasObjectFileName}.</li>
 * <li><b>Temporary file {@link #mtasTmpDocsFileName} with extension
 * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_TMP_DOCS_EXTENSION}</b> <br>
 * Contains for a specific field for each doc multiple fragments. Each occurring
 * term results in a fragment. Structure of content:
 * <ul>
 * <li><b>VInt</b>: docId</li>
 * <li><b>VInt</b>: number of objects in this fragment</li>
 * <li><b>VLong</b>: offset references to {@link #mtasTmpObjectFileName}</li>
 * <li><b>VInt</b>,<b>VLong</b>: mtasId object, reference temporary object in
 * {@link #mtasTmpObjectFileName} minus offset</li>
 * <li><b>VInt</b>,<b>VLong</b>: ...</li>
 * </ul>
 * </li>
 * <li><b>Temporary file {@link #mtasTmpDocsChainedFileName} with extension
 * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_TMP_DOCS_CHAINED_EXTENSION}
 * </b><br>
 * Contains for a specific field for each doc multiple chained fragments.
 * Structure of content:
 * <ul>
 * <li><b>VInt</b>: docId</li>
 * <li><b>VInt</b>: number of objects in this fragment</li>
 * <li><b>VLong</b>: offset references to {@link #mtasTmpObjectFileName}</li>
 * <li><b>VInt</b>,<b>VLong</b>: mtasId object, reference temporary object in
 * {@link #mtasTmpObjectFileName} minus offset</li>
 * <li><b>VInt</b>,<b>VLong</b>: ...</li>
 * <li><b>VLong</b>: reference to next fragment in
 * {@link #mtasTmpDocsChainedFileName}, self reference indicates end of chain
 * </ul>
 * </li>
 * <li><b>Temporary file {@link #mtasTmpDocFileName} with extension
 * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_TMP_DOC_EXTENSION}</b><br>
 * For each document
 * <ul>
 * <li><b>VInt</b>: docId</li>
 * <li><b>VLong</b>: reference to {@link #mtasIndexObjectIdFileName}</li>
 * <li><b>VLong</b>: reference first object, used as offset for tree index
 * <li><b>VInt</b>: slope used in approximation reference objects index on id
 * </li>
 * <li><b>ZLong</b>: offset used in approximation reference objects index on id
 * </li>
 * <li><b>Byte</b>: flag indicating how corrections on the approximation
 * references objects for the index on id are stored:
 * {@link MtasCodecPostingsFormat#MTAS_STORAGE_BYTE},
 * {@link MtasCodecPostingsFormat#MTAS_STORAGE_SHORT},
 * {@link MtasCodecPostingsFormat#MTAS_STORAGE_INTEGER} or
 * {@link MtasCodecPostingsFormat#MTAS_STORAGE_LONG}</li>
 * <li><b>VInt</b>: number of objects in this document</li>
 * <li><b>VInt</b>: first position</li>
 * <li><b>VInt</b>: last position</li>
 * </ul>
 * </li>
 * </ul>
 * </li>
 * <li><b>Final files</b><br>
 * <ul>
 * <li><b>File {@link #mtasIndexFieldFileName} with extension
 * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_FIELD_EXTENSION}</b><br>
 * Contains for each field a reference to the list of documents and the
 * prefixes. Structure of content:
 * <ul>
 * <li><b>String</b>: field</li>
 * <li><b>VLong</b>: reference to {@link #mtasDocFileName}</li>
 * <li><b>VLong</b>: reference to {@link #mtasIndexDocIdFileName}</li>
 * <li><b>VInt</b>: number of documents</li>
 * <li><b>VLong</b>: reference to {@link #mtasTermFileName}</li>
 * <li><b>VInt</b>: number of terms</li>
 * <li><b>VLong</b>: reference to {@link #mtasPrefixFileName}</li>
 * <li><b>VInt</b>: number of prefixes</li>
 * </ul>
 * </li>
 * <li><b>File {@link #mtasTermFileName} with extension
 * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_TERM_EXTENSION}</b><br>
 * For each field, all unique terms are stored here. Structure of content:
 * <ul>
 * <li><b>String</b>: term</li>
 * </ul>
 * </li>
 * <li><b>File {@link #mtasPrefixFileName} with extension
 * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_PREFIX_EXTENSION}</b><br>
 * For each field, all unique prefixes are stored here. Structure of content:
 * <ul>
 * <li><b>String</b>: prefix</li>
 * </ul>
 * </li>
 * <li><b>File {@link #mtasObjectFileName} with extension
 * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_OBJECT_EXTENSION}</b><br>
 * Contains all objects for all fields. Structure of content:
 * <ul>
 * <li><b>VInt</b>: mtasId</li>
 * <li><b>VInt</b>: objectFlags
 * <ul>
 * <li>{@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_PARENT}</li>
 * <li>{@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_POSITION_RANGE}</li>
 * <li>{@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_POSITION_SET}</li>
 * <li>{@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_OFFSET}</li>
 * <li>{@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_REALOFFSET}</li>
 * <li>{@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_PAYLOAD}</li>
 * </ul>
 * </li>
 * <li>Only if {@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_PARENT}<br>
 * <b>VInt</b>: parentId
 * <li>Only if
 * {@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_POSITION_RANGE}<br>
 * <b>VInt</b>,<b>VInt</b>: startPosition and (endPosition-startPosition)
 * <li>Only if {@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_POSITION_SET}<br>
 * <b>VInt</b>,<b>VInt</b>,<b>VInt</b>,...: number of positions, firstPosition,
 * (position-previousPosition),...
 * <li>Only if no {@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_POSITION_RANGE}
 * or {@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_POSITION_SET}<br>
 * <b>VInt</b>: position
 * <li>Only if {@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_OFFSET}<br>
 * <b>VInt</b>,<b>VInt</b>: startOffset, (endOffset-startOffset)
 * <li>Only if {@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_REALOFFSET}<br>
 * <b>VInt</b>,<b>VInt</b>: startRealOffset, (endRealOffset-startRealOffset)
 * <li>Only if {@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_PAYLOAD}<br>
 * <b>VInt</b>,<b>Bytes</b>: number of bytes, payload
 * <li><b>VLong</b>: reference to Term in {@link #mtasTermFileName}</li>
 * </ul>
 * </li>
 * <li><b>File {@link #mtasIndexDocIdFileName} with extension
 * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_INDEX_DOC_ID_EXTENSION}
 * </b><br>
 * Contains for each field a tree structure {@link MtasTree} to search reference
 * to {@link #mtasDocFileName} by id. Structure of content for each node:
 * <ul>
 * <li><b>VLong</b>: offset references to {@link #mtasIndexDocIdFileName}, only
 * available in root node</li>
 * <li><b>Byte</b>: flag, should be zero for this tree, only available in root
 * node</li>
 * <li><b>VInt</b>: left</li>
 * <li><b>VInt</b>: right</li>
 * <li><b>VInt</b>: max</li>
 * <li><b>VLong</b>: left reference to {@link #mtasIndexDocIdFileName} minus the
 * offset stored in the root node</li>
 * <li><b>VLong</b>: right reference to {@link #mtasIndexDocIdFileName} minus
 * the offset stored in the root node</li>
 * <li><b>VInt</b>: number of objects on this node (always 1 for this tree)</li>
 * <li><b>VLong</b>: reference to {@link #mtasDocFileName} minus offset</li>
 * </ul>
 * </li>
 * <li><b>File {@link #mtasDocFileName} with extension
 * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_DOC_EXTENSION}</b><br>
 * For each document
 * <ul>
 * <li><b>VInt</b>: docId</li>
 * <li><b>VLong</b>: reference to {@link #mtasIndexObjectIdFileName}</li>
 * <li><b>VLong</b>: reference to {@link #mtasIndexObjectPositionFileName}</li>
 * <li><b>VLong</b>: reference to {@link #mtasIndexObjectParentFileName}</li>
 * <li><b>VLong</b>: reference first object, used as offset for tree index
 * <li><b>VInt</b>: slope used in approximation reference objects index on id
 * </li>
 * <li><b>ZLong</b>: offset used in approximation reference objects index on id
 * </li>
 * <li><b>Byte</b>: flag indicating how corrections on the approximation
 * references objects for the index on id are stored:
 * {@link MtasCodecPostingsFormat#MTAS_STORAGE_BYTE},
 * {@link MtasCodecPostingsFormat#MTAS_STORAGE_SHORT},
 * {@link MtasCodecPostingsFormat#MTAS_STORAGE_INTEGER} or
 * {@link MtasCodecPostingsFormat#MTAS_STORAGE_LONG}</li>
 * <li><b>VInt</b>: number of objects</li>
 * <li><b>VInt</b>: first position</li>
 * <li><b>VInt</b>: last position</li>
 * </ul>
 * </li>
 * <li><b>File {@link #mtasIndexObjectIdFileName} with extension
 * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_INDEX_OBJECT_ID_EXTENSION}
 * </b><br>
 * Provides for each mtasId the reference to {@link #mtasObjectFileName}. These
 * references are grouped by document, sorted by mtasId, and because the
 * mtasId's for each document will always start with 0 and are sequential
 * without gaps, a reference can be computed if the position of the first
 * reference for a document is known from {@link #mtasDocFileName}. The
 * reference is approximated by the reference to the first object plus the
 * mtasId times a slope. Only a correction to this approximation is stored.
 * Structure of content:
 * <ul>
 * <li><b>Byte</b>/<b>Short</b>/<b>Int</b>/<b>Long</b>: correction reference to
 * {@link #mtasObjectFileName}</li>
 * </ul>
 * </li>
 * <li><b>File {@link #mtasIndexObjectPositionFileName} with extension
 * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_INDEX_OBJECT_POSITION_EXTENSION}
 * </b><br>
 * Contains for each document a tree structure {@link MtasTree} to search
 * objects by position. Structure of content for each node:
 * <ul>
 * <li><b>VLong</b>: offset references to
 * {@link #mtasIndexObjectPositionFileName}, only available in root node</li>
 * <li><b>Byte</b>: flag, should be zero for this tree, only available in root
 * node</li>
 * <li><b>VInt</b>: left</li>
 * <li><b>VInt</b>: right</li>
 * <li><b>VInt</b>: max</li>
 * <li><b>VLong</b>: left reference to {@link #mtasIndexObjectPositionFileName}
 * minus the offset stored in the root node</li>
 * <li><b>VLong</b>: right reference to {@link #mtasIndexObjectPositionFileName}
 * minus the offset stored in the root node</li>
 * <li><b>VInt</b>: number of objects on this node</li>
 * <li><b>VLong</b>,<b>VInt</b>,<b>VLong</b>: set of the first reference to
 * {@link #mtasObjectFileName} minus offset, the prefixId referring to the
 * position the prefix in {@link #mtasPrefixFileName} and the reference to
 * {@link #mtasTermFileName} minus offset</li>
 * <li><b>VLong</b>,<b>VInt</b>,<b>VLong</b>,...: for optional other sets of
 * reference to {@link #mtasObjectFileName}, position of the prefix in
 * {@link #mtasPrefixFileName} and the reference to {@link #mtasTermFileName};
 * for the first item the difference between this reference minus the previous
 * reference is stored</li>
 * </ul>
 * </li>
 * <li><b>File {@link #mtasIndexObjectParentFileName} with extension
 * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_INDEX_OBJECT_PARENT_EXTENSION}
 * </b><br>
 * Contains for each document a tree structure {@link MtasTree} to search
 * objects by parent. Structure of content for each node:
 * <ul>
 * <li><b>VLong</b>: offset references to {@link #mtasIndexObjectParentFileName}
 * , only available in root node</li>
 * <li><b>Byte</b>: flag, for this tree equal to
 * {@link mtas.codec.tree.MtasTree#SINGLE_POSITION_TREE} indicating a tree with
 * exactly one point at each node, only available in root node</li>
 * <li><b>VInt</b>: left</li>
 * <li><b>VInt</b>: right</li>
 * <li><b>VInt</b>: max</li>
 * <li><b>VLong</b>: left reference to {@link #mtasIndexObjectParentFileName}
 * minus the offset stored in the root node</li>
 * <li><b>VLong</b>: right reference to {@link #mtasIndexObjectParentFileName}
 * minus the offset stored in the root node</li>
 * <li><b>VInt</b>: number of objects on this node</li>
 * <li><b>VLong</b>,<b>VInt</b>,<b>VLong</b>: set of the first reference to
 * {@link #mtasObjectFileName} minus offset, the prefixId referring to the
 * position the prefix in {@link #mtasPrefixFileName} and the reference to
 * {@link #mtasTermFileName} minus offset</li>
 * <li><b>VLong</b>,<b>VInt</b>,<b>VLong</b>,...: for optional other sets of
 * reference to {@link #mtasObjectFileName}, position of the prefix in
 * {@link #mtasPrefixFileName} and the reference to {@link #mtasTermFileName};
 * for the first item the difference between this reference minus the previous
 * reference is stored</li>
 * </ul>
 * </li>
 * </ul>
 * </li>
 * </ul>
 * 
 */
public class MtasFieldsConsumer extends FieldsConsumer {

  /** The Constant log. */
  private static final Log log = LogFactory.getLog(MtasFieldsConsumer.class);

  /** The delegate fields consumer. */
  private FieldsConsumer delegateFieldsConsumer;

  /** The state. */
  private SegmentWriteState state;

  /** The intersecting prefixes. */
  private HashMap<String, HashSet<String>> intersectingPrefixes;

  /** The single position prefix. */
  private HashMap<String, HashSet<String>> singlePositionPrefix;

  /** The multiple position prefix. */
  private HashMap<String, HashSet<String>> multiplePositionPrefix;

  /** The set position prefix. */
  private HashMap<String, HashSet<String>> setPositionPrefix;

  /** The prefix reference index. */
  private HashMap<String, HashMap<String, Long>> prefixReferenceIndex;

  /** The prefix id index. */
  private HashMap<String, HashMap<String, Integer>> prefixIdIndex;

  /** The token stats min pos. */
  Integer tokenStatsMinPos;

  /** The token stats max pos. */
  Integer tokenStatsMaxPos;

  /** The token stats number. */
  Integer tokenStatsNumber;

  /** The mtas tmp field file name. */
  private String mtasTmpFieldFileName;

  /** The mtas tmp object file name. */
  private String mtasTmpObjectFileName;

  /** The mtas tmp docs file name. */
  private String mtasTmpDocsFileName;

  /** The mtas tmp doc file name. */
  private String mtasTmpDocFileName;

  /** The mtas tmp docs chained file name. */
  private String mtasTmpDocsChainedFileName;

  /** The mtas object file name. */
  private String mtasObjectFileName;

  /** The mtas term file name. */
  private String mtasTermFileName;

  /** The mtas index field file name. */
  private String mtasIndexFieldFileName;

  /** The mtas prefix file name. */
  private String mtasPrefixFileName;

  /** The mtas doc file name. */
  private String mtasDocFileName;

  /** The mtas index doc id file name. */
  private String mtasIndexDocIdFileName;

  /** The mtas index object id file name. */
  private String mtasIndexObjectIdFileName;

  /** The mtas index object position file name. */
  private String mtasIndexObjectPositionFileName;

  /** The mtas index object parent file name. */
  private String mtasIndexObjectParentFileName;

  /** The name. */
  private String name;

  /** The delegate postings format name. */
  private String delegatePostingsFormatName;

  /**
   * Instantiates a new mtas fields consumer.
   *
   * @param fieldsConsumer
   *          the fields consumer
   * @param state
   *          the state
   * @param name
   *          the name
   * @param delegatePostingsFormatName
   *          the delegate postings format name
   */
  public MtasFieldsConsumer(FieldsConsumer fieldsConsumer,
      SegmentWriteState state, String name, String delegatePostingsFormatName) {
    this.delegateFieldsConsumer = fieldsConsumer;
    this.state = state;
    this.name = name;
    this.delegatePostingsFormatName = delegatePostingsFormatName;
    // temporary fileNames
    mtasTmpFieldFileName = IndexFileNames.segmentFileName(
        state.segmentInfo.name, state.segmentSuffix,
        MtasCodecPostingsFormat.MTAS_TMP_FIELD_EXTENSION);
    mtasTmpObjectFileName = IndexFileNames.segmentFileName(
        state.segmentInfo.name, state.segmentSuffix,
        MtasCodecPostingsFormat.MTAS_TMP_OBJECT_EXTENSION);
    mtasTmpDocsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
        state.segmentSuffix, MtasCodecPostingsFormat.MTAS_TMP_DOCS_EXTENSION);
    mtasTmpDocFileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
        state.segmentSuffix, MtasCodecPostingsFormat.MTAS_TMP_DOC_EXTENSION);
    mtasTmpDocsChainedFileName = IndexFileNames.segmentFileName(
        state.segmentInfo.name, state.segmentSuffix,
        MtasCodecPostingsFormat.MTAS_TMP_DOCS_CHAINED_EXTENSION);
    // fileNames
    mtasObjectFileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
        state.segmentSuffix, MtasCodecPostingsFormat.MTAS_OBJECT_EXTENSION);
    mtasTermFileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
        state.segmentSuffix, MtasCodecPostingsFormat.MTAS_TERM_EXTENSION);
    mtasIndexFieldFileName = IndexFileNames.segmentFileName(
        state.segmentInfo.name, state.segmentSuffix,
        MtasCodecPostingsFormat.MTAS_FIELD_EXTENSION);
    mtasPrefixFileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
        state.segmentSuffix, MtasCodecPostingsFormat.MTAS_PREFIX_EXTENSION);
    mtasDocFileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
        state.segmentSuffix, MtasCodecPostingsFormat.MTAS_DOC_EXTENSION);
    mtasIndexDocIdFileName = IndexFileNames.segmentFileName(
        state.segmentInfo.name, state.segmentSuffix,
        MtasCodecPostingsFormat.MTAS_INDEX_DOC_ID_EXTENSION);
    mtasIndexObjectIdFileName = IndexFileNames.segmentFileName(
        state.segmentInfo.name, state.segmentSuffix,
        MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_ID_EXTENSION);
    mtasIndexObjectPositionFileName = IndexFileNames.segmentFileName(
        state.segmentInfo.name, state.segmentSuffix,
        MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_POSITION_EXTENSION);
    mtasIndexObjectParentFileName = IndexFileNames.segmentFileName(
        state.segmentInfo.name, state.segmentSuffix,
        MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_PARENT_EXTENSION);
  }

  /**
   * Register prefix.
   *
   * @param field
   *          the field
   * @param prefix
   *          the prefix
   * @param outPrefix
   *          the out prefix
   * @throws IOException
   *           Signals that an I/O exception has occurred.
   */
  private void registerPrefix(String field, String prefix,
      IndexOutput outPrefix) throws IOException {
    if (!prefixReferenceIndex.containsKey(field)) {
      prefixReferenceIndex.put(field, new HashMap<String, Long>());
      prefixIdIndex.put(field, new HashMap<String, Integer>());
    }
    if (!prefixReferenceIndex.get(field).containsKey(prefix)) {
      int id = 1 + prefixReferenceIndex.get(field).size();
      prefixReferenceIndex.get(field).put(prefix, outPrefix.getFilePointer());
      prefixIdIndex.get(field).put(prefix, id);
      outPrefix.writeString(prefix);
    }
  }

  /**
   * Register prefix intersection.
   *
   * @param field
   *          the field
   * @param prefix
   *          the prefix
   * @param start
   *          the start
   * @param end
   *          the end
   * @param docFieldAdministration
   *          the doc field administration
   */
  private void registerPrefixIntersection(String field, String prefix,
      int start, int end,
      HashMap<String, HashSet<Integer>> docFieldAdministration) {
    if (!intersectingPrefixes.containsKey(field)) {
      intersectingPrefixes.put(field, new HashSet<String>());
    } else if (intersectingPrefixes.get(field).contains(prefix)) {
      return;
    }
    HashSet<Integer> docFieldPrefixAdministration;
    if (!docFieldAdministration.containsKey(prefix)) {
      docFieldPrefixAdministration = new HashSet<>();
      docFieldAdministration.put(prefix, docFieldPrefixAdministration);
    } else {
      docFieldPrefixAdministration = docFieldAdministration.get(prefix);
      // check
      for (int p = start; p <= end; p++) {
        if (docFieldPrefixAdministration.contains(p)) {
          intersectingPrefixes.get(field).add(prefix);
          docFieldAdministration.remove(prefix);
          return;
        }
      }
    }
    // update
    for (int p = start; p <= end; p++) {
      docFieldPrefixAdministration.add(p);
    }
  }

  /**
   * Register prefix stats single position value.
   *
   * @param field
   *          the field
   * @param prefix
   *          the prefix
   * @param outPrefix
   *          the out prefix
   * @throws IOException
   *           Signals that an I/O exception has occurred.
   */
  public void registerPrefixStatsSinglePositionValue(String field,
      String prefix, IndexOutput outPrefix) throws IOException {
    initPrefixStatsField(field);
    registerPrefix(field, prefix, outPrefix);
    if (!multiplePositionPrefix.get(field).contains(prefix)) {
      singlePositionPrefix.get(field).add(prefix);
    }
  }

  /**
   * Register prefix stats range position value.
   *
   * @param field
   *          the field
   * @param prefix
   *          the prefix
   * @param outPrefix
   *          the out prefix
   * @throws IOException
   *           Signals that an I/O exception has occurred.
   */
  public void registerPrefixStatsRangePositionValue(String field, String prefix,
      IndexOutput outPrefix) throws IOException {
    initPrefixStatsField(field);
    registerPrefix(field, prefix, outPrefix);
    singlePositionPrefix.get(field).remove(prefix);
    multiplePositionPrefix.get(field).add(prefix);
  }

  /**
   * Register prefix stats set position value.
   *
   * @param field
   *          the field
   * @param prefix
   *          the prefix
   * @param outPrefix
   *          the out prefix
   * @throws IOException
   *           Signals that an I/O exception has occurred.
   */
  public void registerPrefixStatsSetPositionValue(String field, String prefix,
      IndexOutput outPrefix) throws IOException {
    initPrefixStatsField(field);
    registerPrefix(field, prefix, outPrefix);
    singlePositionPrefix.get(field).remove(prefix);
    multiplePositionPrefix.get(field).add(prefix);
    setPositionPrefix.get(field).add(prefix);
  }

  /**
   * Inits the prefix stats field.
   *
   * @param field
   *          the field
   */
  private void initPrefixStatsField(String field) {
    if (!singlePositionPrefix.containsKey(field)) {
      singlePositionPrefix.put(field, new HashSet<String>());
    }
    if (!multiplePositionPrefix.containsKey(field)) {
      multiplePositionPrefix.put(field, new HashSet<String>());
    }
    if (!setPositionPrefix.containsKey(field)) {
      setPositionPrefix.put(field, new HashSet<String>());
    }
  }

  /**
   * Gets the prefix stats single position prefix attribute.
   *
   * @param field
   *          the field
   * @return the prefix stats single position prefix attribute
   */
  public String getPrefixStatsSinglePositionPrefixAttribute(String field) {
    return String.join(MtasToken.DELIMITER, singlePositionPrefix.get(field));
  }

  /**
   * Gets the prefix stats multiple position prefix attribute.
   *
   * @param field
   *          the field
   * @return the prefix stats multiple position prefix attribute
   */
  public String getPrefixStatsMultiplePositionPrefixAttribute(String field) {
    return String.join(MtasToken.DELIMITER, multiplePositionPrefix.get(field));
  }

  /**
   * Gets the prefix stats set position prefix attribute.
   *
   * @param field
   *          the field
   * @return the prefix stats set position prefix attribute
   */
  public String getPrefixStatsSetPositionPrefixAttribute(String field) {
    return String.join(MtasToken.DELIMITER, setPositionPrefix.get(field));
  }

  /**
   * Gets the prefix stats intersection prefix attribute.
   *
   * @param field
   *          the field
   * @return the prefix stats intersection prefix attribute
   */
  public String getPrefixStatsIntersectionPrefixAttribute(String field) {
    if (intersectingPrefixes.containsKey(field)) {
      return String.join(MtasToken.DELIMITER, intersectingPrefixes.get(field));
    } else {
      return "";
    }
  }

  /*
   * (non-Javadoc)
   * 
   * @see org.apache.lucene.codecs.FieldsConsumer#merge(org.apache.lucene.index.
   * MergeState)
   */
  @Override
  public void merge(MergeState mergeState) throws IOException {
    final List<Fields> fields = new ArrayList<>();
    final List<ReaderSlice> slices = new ArrayList<>();

    int docBase = 0;

    for (int readerIndex = 0; readerIndex < mergeState.fieldsProducers.length; readerIndex++) {
      final FieldsProducer f = mergeState.fieldsProducers[readerIndex];

      final int maxDoc = mergeState.maxDocs[readerIndex];
      f.checkIntegrity();
      slices.add(new ReaderSlice(docBase, maxDoc, readerIndex));
      fields.add(f);
      docBase += maxDoc;
    }

    Fields mergedFields = new MappedMultiFields(mergeState,
        new MultiFields(fields.toArray(Fields.EMPTY_ARRAY),
            slices.toArray(ReaderSlice.EMPTY_ARRAY)));
    write(mergedFields);
  }

  /*
   * (non-Javadoc)
   * 
   * @see org.apache.lucene.codecs.FieldsConsumer#write(org.apache.lucene.index.
   * Fields )
   */
  @Override
  public void write(Fields fields) throws IOException {
    delegateFieldsConsumer.write(fields);
    write(state.fieldInfos, fields);
  }

  /**
   * Write.
   *
   * @param fieldInfos
   *          the field infos
   * @param fields
   *          the fields
   */
  private void write(FieldInfos fieldInfos, Fields fields) {
    IndexOutput outField;
    IndexOutput outDoc;
    IndexOutput outIndexDocId;
    IndexOutput outIndexObjectId;
    IndexOutput outIndexObjectPosition;
    IndexOutput outIndexObjectParent;
    IndexOutput outTerm;
    IndexOutput outObject;
    IndexOutput outPrefix;
    IndexOutput outTmpDoc;
    IndexOutput outTmpField;
    HashSet<Closeable> closeables = new HashSet<>();
    // prefix stats
    intersectingPrefixes = new HashMap<>();
    singlePositionPrefix = new HashMap<>();
    multiplePositionPrefix = new HashMap<>();
    setPositionPrefix = new HashMap<>();
    prefixReferenceIndex = new HashMap<>();
    prefixIdIndex = new HashMap<>();
    // temporary temporary index in memory for doc
    SortedMap<Integer, Long> memoryIndexTemporaryObject = new TreeMap<>();
    // create (backwards) chained new temporary index docs
    SortedMap<Integer, Long> memoryTmpDocChainList = new TreeMap<>();
    // list of objectIds and references to objects
    SortedMap<Integer, Long> memoryIndexDocList = new TreeMap<>();

    try {
      // create file tmpDoc
      closeables.add(outTmpDoc = state.directory
          .createOutput(mtasTmpDocFileName, state.context));
      // create file tmpField
      closeables.add(outTmpField = state.directory
          .createOutput(mtasTmpFieldFileName, state.context));
      // create file indexDoc
      closeables.add(outDoc = state.directory.createOutput(mtasDocFileName,
          state.context));
      CodecUtil.writeIndexHeader(outDoc, name,
          MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
          state.segmentSuffix);
      outDoc.writeString(delegatePostingsFormatName);
      // create file indexDocId
      closeables.add(outIndexDocId = state.directory
          .createOutput(mtasIndexDocIdFileName, state.context));
      CodecUtil.writeIndexHeader(outIndexDocId, name,
          MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
          state.segmentSuffix);
      outIndexDocId.writeString(delegatePostingsFormatName);
      // create file indexObjectId
      closeables.add(outIndexObjectId = state.directory
          .createOutput(mtasIndexObjectIdFileName, state.context));
      CodecUtil.writeIndexHeader(outIndexObjectId, name,
          MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
          state.segmentSuffix);
      outIndexObjectId.writeString(delegatePostingsFormatName);
      // create file indexObjectPosition
      closeables.add(outIndexObjectPosition = state.directory
          .createOutput(mtasIndexObjectPositionFileName, state.context));
      CodecUtil.writeIndexHeader(outIndexObjectPosition, name,
          MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
          state.segmentSuffix);
      outIndexObjectPosition.writeString(delegatePostingsFormatName);
      // create file indexObjectParent
      closeables.add(outIndexObjectParent = state.directory
          .createOutput(mtasIndexObjectParentFileName, state.context));
      CodecUtil.writeIndexHeader(outIndexObjectParent, name,
          MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
          state.segmentSuffix);
      outIndexObjectParent.writeString(delegatePostingsFormatName);
      // create file term
      closeables.add(outTerm = state.directory.createOutput(mtasTermFileName,
          state.context));
      CodecUtil.writeIndexHeader(outTerm, name,
          MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
          state.segmentSuffix);
      outTerm.writeString(delegatePostingsFormatName);
      // create file prefix
      closeables.add(outPrefix = state.directory
          .createOutput(mtasPrefixFileName, state.context));
      CodecUtil.writeIndexHeader(outPrefix, name,
          MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
          state.segmentSuffix);
      outPrefix.writeString(delegatePostingsFormatName);
      // create file object
      closeables.add(outObject = state.directory
          .createOutput(mtasObjectFileName, state.context));
      CodecUtil.writeIndexHeader(outObject, name,
          MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
          state.segmentSuffix);
      outObject.writeString(delegatePostingsFormatName);
      // For each field
      for (String field : fields) {
        Terms terms = fields.terms(field);
        if (terms == null) {
          continue;
        } else {
          // new temporary object storage for this field
          IndexOutput outTmpObject = state.directory
              .createOutput(mtasTmpObjectFileName, state.context);
          closeables.add(outTmpObject);
          // new temporary index docs for this field
          IndexOutput outTmpDocs = state.directory
              .createOutput(mtasTmpDocsFileName, state.context);
          closeables.add(outTmpDocs);
          // get fieldInfo
          FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
          // get properties terms
          boolean hasPositions = terms.hasPositions();
          boolean hasFreqs = terms.hasFreqs();
          boolean hasPayloads = fieldInfo.hasPayloads();
          boolean hasOffsets = terms.hasOffsets();
          // register references
          Long smallestTermFilepointer = outTerm.getFilePointer();
          Long smallestPrefixFilepointer = outPrefix.getFilePointer();
          int termCounter = 0;
          // only if freqs, positions and payload available
          if (hasFreqs && hasPositions && hasPayloads) {
            // compute flags
            int flags = PostingsEnum.POSITIONS | PostingsEnum.PAYLOADS;
            if (hasOffsets) {
              flags = flags | PostingsEnum.OFFSETS;
            }
            // get terms
            TermsEnum termsEnum = terms.iterator();
            PostingsEnum postingsEnum = null;
            // for each term in field
            while (true) {
              BytesRef term = termsEnum.next();
              if (term == null) {
                break;
              }
              // store term and get ref
              Long termRef = outTerm.getFilePointer();
              outTerm.writeString(term.utf8ToString());
              termCounter++;
              // get postings
              postingsEnum = termsEnum.postings(postingsEnum, flags);
              // for each doc in field+term
              while (true) {
                Integer docId = postingsEnum.nextDoc();
                if (docId.equals(DocIdSetIterator.NO_MORE_DOCS)) {
                  break;
                }
                int freq = postingsEnum.freq();
                // temporary storage objects and temporary index in memory for
                // doc
                memoryIndexTemporaryObject.clear();
                Long offsetFilePointerTmpObject = outTmpObject.getFilePointer();
                for (int i = 0; i < freq; i++) {
                  Long currentFilePointerTmpObject = outTmpObject
                      .getFilePointer();
                  Integer mtasId;
                  int position = postingsEnum.nextPosition();
                  BytesRef payload = postingsEnum.getPayload();
                  if (hasOffsets) {
                    mtasId = createObjectAndRegisterPrefix(field, outTmpObject,
                        term, termRef, position, payload,
                        postingsEnum.startOffset(), postingsEnum.endOffset(),
                        outPrefix);
                  } else {
                    mtasId = createObjectAndRegisterPrefix(field, outTmpObject,
                        term, termRef, position, payload, outPrefix);
                  }
                  if (mtasId != null) {
                    assert !memoryIndexTemporaryObject.containsKey(
                        mtasId) : "mtasId should be unique in this selection";
                    memoryIndexTemporaryObject.put(mtasId,
                        currentFilePointerTmpObject);
                  }
                } // end loop positions
                // store temporary index for this doc
                if (memoryIndexTemporaryObject.size() > 0) {
                  // docId for this part
                  outTmpDocs.writeVInt(docId);
                  // number of objects/tokens in this part
                  outTmpDocs.writeVInt(memoryIndexTemporaryObject.size());
                  // offset to be used for references
                  outTmpDocs.writeVLong(offsetFilePointerTmpObject);
                  // loop over tokens
                  for (Entry<Integer, Long> entry : memoryIndexTemporaryObject
                      .entrySet()) {
                    // mtasId object
                    outTmpDocs.writeVInt(entry.getKey());
                    // reference object
                    outTmpDocs.writeVLong(
                        (entry.getValue() - offsetFilePointerTmpObject));
                  }
                }
                // clean up
                memoryIndexTemporaryObject.clear();
              } // end loop docs
            } // end loop terms
            // set fieldInfo
            fieldInfos.fieldInfo(field).putAttribute(
                MtasCodecPostingsFormat.MTAS_FIELDINFO_ATTRIBUTE_PREFIX_SINGLE_POSITION,
                getPrefixStatsSinglePositionPrefixAttribute(field));
            fieldInfos.fieldInfo(field).putAttribute(
                MtasCodecPostingsFormat.MTAS_FIELDINFO_ATTRIBUTE_PREFIX_MULTIPLE_POSITION,
                getPrefixStatsMultiplePositionPrefixAttribute(field));
            fieldInfos.fieldInfo(field).putAttribute(
                MtasCodecPostingsFormat.MTAS_FIELDINFO_ATTRIBUTE_PREFIX_SET_POSITION,
                getPrefixStatsSetPositionPrefixAttribute(field));
          } // end processing field with freqs, positions and payload
          // close temporary object storage and index docs
          outTmpObject.close();
          closeables.remove(outTmpObject);
          outTmpDocs.close();
          closeables.remove(outTmpDocs);

          // create (backwards) chained new temporary index docs
          IndexInput inTmpDocs = state.directory.openInput(mtasTmpDocsFileName,
              state.context);
          closeables.add(inTmpDocs);
          IndexOutput outTmpDocsChained = state.directory
              .createOutput(mtasTmpDocsChainedFileName, state.context);
          closeables.add(outTmpDocsChained);
          memoryTmpDocChainList.clear();
          while (true) {
            try {
              Long currentFilepointer = outTmpDocsChained.getFilePointer();
              // copy docId
              int docId = inTmpDocs.readVInt();
              outTmpDocsChained.writeVInt(docId);
              // copy size
              int size = inTmpDocs.readVInt();
              outTmpDocsChained.writeVInt(size);
              // offset references
              outTmpDocsChained.writeVLong(inTmpDocs.readVLong());
              for (int t = 0; t < size; t++) {
                outTmpDocsChained.writeVInt(inTmpDocs.readVInt());
                outTmpDocsChained.writeVLong(inTmpDocs.readVLong());
              }
              // set back reference to part with same docId
              if (memoryTmpDocChainList.containsKey(docId)) {
                // reference to previous
                outTmpDocsChained.writeVLong(memoryTmpDocChainList.get(docId));
              } else {
                // self reference indicates end of chain
                outTmpDocsChained.writeVLong(currentFilepointer);
              }
              // update temporary index in memory
              memoryTmpDocChainList.put(docId, currentFilepointer);
            } catch (IOException ex) {
              log.debug(ex);
              break;
            }
          }
          outTmpDocsChained.close();
          closeables.remove(outTmpDocsChained);
          inTmpDocs.close();
          closeables.remove(inTmpDocs);
          state.directory.deleteFile(mtasTmpDocsFileName);

          // set reference to tmpDoc in Field
          if (memoryTmpDocChainList.size() > 0) {
            outTmpField.writeString(field);
            outTmpField.writeVLong(outTmpDoc.getFilePointer());
            outTmpField.writeVInt(memoryTmpDocChainList.size());
            outTmpField.writeVLong(smallestTermFilepointer);
            outTmpField.writeVInt(termCounter);
            outTmpField.writeVLong(smallestPrefixFilepointer);
            outTmpField.writeVInt(prefixReferenceIndex.get(field).size());
            // fill indexDoc
            IndexInput inTmpDocsChained = state.directory
                .openInput(mtasTmpDocsChainedFileName, state.context);
            closeables.add(inTmpDocsChained);
            IndexInput inTmpObject = state.directory
                .openInput(mtasTmpObjectFileName, state.context);
            closeables.add(inTmpObject);
            for (Entry<Integer, Long> entry : memoryTmpDocChainList
                .entrySet()) {
              Integer docId = entry.getKey();
              Long currentFilePointer;
              Long newFilePointer;
              // list of objectIds and references to objects
              memoryIndexDocList.clear();
              // construct final object + indexObjectId for docId
              currentFilePointer = entry.getValue();
              // collect objects for document
              tokenStatsMinPos = null;
              tokenStatsMaxPos = null;
              tokenStatsNumber = 0;
              while (true) {
                inTmpDocsChained.seek(currentFilePointer);
                Integer docIdPart = inTmpDocsChained.readVInt();
                assert docIdPart.equals(
                    docId) : "conflicting docId in reference to temporaryIndexDocsChained";
                // number of objects/tokens in part
                int size = inTmpDocsChained.readVInt();
                long offsetFilePointerTmpObject = inTmpDocsChained.readVLong();
                assert size > 0 : "number of objects/tokens in part cannot be "
                    + size;
                for (int t = 0; t < size; t++) {
                  int mtasId = inTmpDocsChained.readVInt();
                  Long tmpObjectRef = inTmpDocsChained.readVLong()
                      + offsetFilePointerTmpObject;
                  assert !memoryIndexDocList.containsKey(
                      mtasId) : "mtasId should be unique in this selection";
                  // initially, store ref to tmpObject
                  memoryIndexDocList.put(mtasId, tmpObjectRef);
                }
                // reference to next part
                newFilePointer = inTmpDocsChained.readVLong();
                if (newFilePointer.equals(currentFilePointer)) {
                  break; // end of chained parts
                } else {
                  currentFilePointer = newFilePointer;
                }
              }
              // now create new objects, sorted by mtasId
              Long smallestObjectFilepointer = outObject.getFilePointer();
              for (Entry<Integer, Long> objectEntry : memoryIndexDocList
                  .entrySet()) {
                int mtasId = objectEntry.getKey();
                Long tmpObjectRef = objectEntry.getValue();
                Long objectRef = outObject.getFilePointer();
                copyObjectAndUpdateStats(mtasId, inTmpObject, tmpObjectRef,
                    outObject);
                // update with new ref
                memoryIndexDocList.put(mtasId, objectRef);
              }
              // check mtasIds properties
              assert memoryIndexDocList.firstKey()
                  .equals(0) : "first mtasId should not be "
                      + memoryIndexDocList.firstKey();
              assert (1 + memoryIndexDocList.lastKey()
                  - memoryIndexDocList.firstKey()) == memoryIndexDocList
                      .size() : "missing mtasId";
              assert tokenStatsNumber.equals(memoryIndexDocList
                  .size()) : "incorrect number of items in tokenStats";

              // store item in tmpDoc
              outTmpDoc.writeVInt(docId);
              outTmpDoc.writeVLong(outIndexObjectId.getFilePointer());

              int mtasId = 0;
              // compute linear approximation (least squares method, integer
              // constants)
              long tmpN = memoryIndexDocList.size();
              long tmpSumY = 0;
              long tmpSumXY = 0;
              long tmpSumX = 0;
              long tmpSumXX = 0;
              for (Entry<Integer, Long> objectEntry : memoryIndexDocList
                  .entrySet()) {
                assert objectEntry.getKey()
                    .equals(mtasId) : "unexpected mtasId";
                tmpSumY += objectEntry.getValue();
                tmpSumX += mtasId;
                tmpSumXY += mtasId * objectEntry.getValue();
                tmpSumXX += mtasId * mtasId;
                mtasId++;
              }
              int objectRefApproxQuotient;
              if(tmpN>1) {
                objectRefApproxQuotient= (int) (((tmpN * tmpSumXY)              
                  - (tmpSumX * tmpSumY))
                  / ((tmpN * tmpSumXX) - (tmpSumX * tmpSumX)));
              } else {
                objectRefApproxQuotient = 0;
              }
              long objectRefApproxOffset = (tmpSumY
                  - objectRefApproxQuotient * tmpSumX) / tmpN;
              Long objectRefApproxCorrection;
              long maxAbsObjectRefApproxCorrection = 0;
              // compute maximum correction
              mtasId = 0;
              for (Entry<Integer, Long> objectEntry : memoryIndexDocList
                  .entrySet()) {
                objectRefApproxCorrection = (objectEntry.getValue()
                    - (objectRefApproxOffset
                        + (mtasId * objectRefApproxQuotient)));
                maxAbsObjectRefApproxCorrection = Math.max(
                    maxAbsObjectRefApproxCorrection,
                    Math.abs(objectRefApproxCorrection));
                mtasId++;
              }
              byte storageFlags;
              if (maxAbsObjectRefApproxCorrection <= Long
                  .valueOf(Byte.MAX_VALUE)) {
                storageFlags = MtasCodecPostingsFormat.MTAS_STORAGE_BYTE;
              } else if (maxAbsObjectRefApproxCorrection <= Long
                  .valueOf(Short.MAX_VALUE)) {
                storageFlags = MtasCodecPostingsFormat.MTAS_STORAGE_SHORT;
              } else if (maxAbsObjectRefApproxCorrection <= Long
                  .valueOf(Integer.MAX_VALUE)) {
                storageFlags = MtasCodecPostingsFormat.MTAS_STORAGE_INTEGER;
              } else {
                storageFlags = MtasCodecPostingsFormat.MTAS_STORAGE_LONG;
              }
              // update indexObjectId with correction on approximated ref
              // (assume
              // can be stored as int)
              mtasId = 0;
              for (Entry<Integer, Long> objectEntry : memoryIndexDocList
                  .entrySet()) {
                objectRefApproxCorrection = (objectEntry.getValue()
                    - (objectRefApproxOffset
                        + (mtasId * objectRefApproxQuotient)));
                if (storageFlags == MtasCodecPostingsFormat.MTAS_STORAGE_BYTE) {
                  outIndexObjectId
                      .writeByte(objectRefApproxCorrection.byteValue());
                } else if (storageFlags == MtasCodecPostingsFormat.MTAS_STORAGE_SHORT) {
                  outIndexObjectId
                      .writeShort(objectRefApproxCorrection.shortValue());
                } else if (storageFlags == MtasCodecPostingsFormat.MTAS_STORAGE_INTEGER) {
                  outIndexObjectId
                      .writeInt(objectRefApproxCorrection.intValue());
                } else {
                  outIndexObjectId.writeLong(objectRefApproxCorrection);
                }
                mtasId++;
              }
              outTmpDoc.writeVLong(smallestObjectFilepointer);
              outTmpDoc.writeVInt(objectRefApproxQuotient);
              outTmpDoc.writeZLong(objectRefApproxOffset);
              outTmpDoc.writeByte(storageFlags);
              outTmpDoc.writeVInt(tokenStatsNumber);
              outTmpDoc.writeVInt(tokenStatsMinPos);
              outTmpDoc.writeVInt(tokenStatsMaxPos);
              // clean up
              memoryIndexDocList.clear();
            } // end loop over docs
            inTmpDocsChained.close();
            closeables.remove(inTmpDocsChained);
            inTmpObject.close();
            closeables.remove(inTmpObject);
          }
          // clean up
          memoryTmpDocChainList.clear();
          // remove temporary files
          state.directory.deleteFile(mtasTmpObjectFileName);
          state.directory.deleteFile(mtasTmpDocsChainedFileName);
          // store references for field

        } // end processing field
      } // end loop fields
      // close temporary index doc
      outTmpDoc.close();
      closeables.remove(outTmpDoc);
      // close indexField, indexObjectId and object
      CodecUtil.writeFooter(outTmpField);
      outTmpField.close();
      closeables.remove(outTmpField);
      CodecUtil.writeFooter(outIndexObjectId);
      outIndexObjectId.close();
      closeables.remove(outIndexObjectId);
      CodecUtil.writeFooter(outObject);
      outObject.close();
      closeables.remove(outObject);
      CodecUtil.writeFooter(outTerm);
      outTerm.close();
      closeables.remove(outTerm);
      CodecUtil.writeFooter(outPrefix);
      outPrefix.close();
      closeables.remove(outPrefix);

      // create final doc, fill indexObjectPosition, indexObjectParent and
      // indexTermPrefixPosition, create final field
      IndexInput inTmpField = state.directory.openInput(mtasTmpFieldFileName,
          state.context);
      closeables.add(inTmpField);
      IndexInput inTmpDoc = state.directory.openInput(mtasTmpDocFileName,
          state.context);
      closeables.add(inTmpDoc);
      IndexInput inObjectId = state.directory
          .openInput(mtasIndexObjectIdFileName, state.context);
      closeables.add(inObjectId);
      IndexInput inObject = state.directory.openInput(mtasObjectFileName,
          state.context);
      closeables.add(inObject);
      IndexInput inTerm = state.directory.openInput(mtasTermFileName,
          state.context);
      closeables.add(inTerm);
      closeables.add(outField = state.directory
          .createOutput(mtasIndexFieldFileName, state.context));
      CodecUtil.writeIndexHeader(outField, name,
          MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
          state.segmentSuffix);
      outField.writeString(delegatePostingsFormatName);
      boolean doWrite = true;
      do {
        try {
          // read from tmpField
          String field = inTmpField.readString();
          long fpTmpDoc = inTmpField.readVLong();
          int numberDocs = inTmpField.readVInt();
          long fpTerm = inTmpField.readVLong();
          int numberTerms = inTmpField.readVInt();
          long fpPrefix = inTmpField.readVLong();
          int numberPrefixes = inTmpField.readVInt();
          inTmpDoc.seek(fpTmpDoc);
          long fpFirstDoc = outDoc.getFilePointer();
          // get prefixId index
          HashMap<String, Integer> prefixIdIndexField = prefixIdIndex
              .get(field);
          // construct MtasRBTree for indexDocId
          MtasRBTree mtasDocIdTree = new MtasRBTree(true, false);
          for (int docCounter = 0; docCounter < numberDocs; docCounter++) {
            // get info from tmpDoc
            int docId = inTmpDoc.readVInt();
            // filePointer indexObjectId
            Long fpIndexObjectId = inTmpDoc.readVLong();
            // filePointer indexObjectPosition (unknown)
            Long fpIndexObjectPosition;
            // filePointer indexObjectParent (unknown)
            Long fpIndexObjectParent;
            // constants for approximation object references for this document
            long smallestObjectFilepointer = inTmpDoc.readVLong();
            int objectRefApproxQuotient = inTmpDoc.readVInt();
            long objectRefApproxOffset = inTmpDoc.readZLong();
            byte storageFlags = inTmpDoc.readByte();
            // number objects/tokens
            int size = inTmpDoc.readVInt();
            // construct MtasRBTree
            MtasRBTree mtasPositionTree = new MtasRBTree(false, true);
            MtasRBTree mtasParentTree = new MtasRBTree(false, true);
            inObjectId.seek(fpIndexObjectId);
            long refCorrection;
            long ref;
            HashMap<String, HashSet<Integer>> docFieldAdministration = new HashMap<>();
            for (int mtasId = 0; mtasId < size; mtasId++) {
              if (storageFlags == MtasCodecPostingsFormat.MTAS_STORAGE_BYTE) {
                refCorrection = inObjectId.readByte();
              } else if (storageFlags == MtasCodecPostingsFormat.MTAS_STORAGE_SHORT) {
                refCorrection = inObjectId.readShort();
              } else if (storageFlags == MtasCodecPostingsFormat.MTAS_STORAGE_INTEGER) {
                refCorrection = inObjectId.readInt();
              } else {
                refCorrection = inObjectId.readLong();
              }
              ref = objectRefApproxOffset + mtasId * objectRefApproxQuotient
                  + refCorrection;
              MtasTokenString token = MtasCodecPostingsFormat.getToken(inObject,
                  inTerm, ref);
              String prefix = token.getPrefix();
              registerPrefixIntersection(field, prefix,
                  token.getPositionStart(), token.getPositionEnd(),
                  docFieldAdministration);
              int prefixId = prefixIdIndexField.containsKey(prefix)
                  ? prefixIdIndexField.get(prefix) : 0;
              token.setPrefixId(prefixId);
              assert token.getId().equals(mtasId) : "unexpected mtasId "
                  + mtasId;
              mtasPositionTree.addPositionAndObjectFromToken(token);
              mtasParentTree.addParentFromToken(token);
            }
            // store mtasPositionTree and mtasParentTree
            fpIndexObjectPosition = storeTree(mtasPositionTree,
                outIndexObjectPosition, smallestObjectFilepointer);
            fpIndexObjectParent = storeTree(mtasParentTree,
                outIndexObjectParent, smallestObjectFilepointer);
            long fpDoc = outDoc.getFilePointer();
            // create indexDoc with updated fpIndexObjectPosition from tmpDoc
            outDoc.writeVInt(docId); // docId
            // reference indexObjectId
            outDoc.writeVLong(fpIndexObjectId);
            // reference indexObjectPosition
            outDoc.writeVLong(fpIndexObjectPosition);
            // reference indexObjectParent
            outDoc.writeVLong(fpIndexObjectParent);
            // variables approximation and storage references object
            outDoc.writeVLong(smallestObjectFilepointer);
            outDoc.writeVInt(objectRefApproxQuotient);
            outDoc.writeZLong(objectRefApproxOffset);
            outDoc.writeByte(storageFlags);
            // number of objects
            outDoc.writeVInt(size);
            // minPosition
            outDoc.writeVInt(inTmpDoc.readVInt());
            // maxPosition
            outDoc.writeVInt(inTmpDoc.readVInt());
            // add to tree for indexDocId
            mtasDocIdTree.addIdFromDoc(docId, fpDoc);
          }
          long fpIndexDocId = storeTree(mtasDocIdTree, outIndexDocId,
              fpFirstDoc);

          // store in indexField
          outField.writeString(field);
          outField.writeVLong(fpFirstDoc);
          outField.writeVLong(fpIndexDocId);
          outField.writeVInt(numberDocs);
          outField.writeVLong(fpTerm);
          outField.writeVInt(numberTerms);
          outField.writeVLong(fpPrefix);
          outField.writeVInt(numberPrefixes);
          // register intersection
          fieldInfos.fieldInfo(field).putAttribute(
              MtasCodecPostingsFormat.MTAS_FIELDINFO_ATTRIBUTE_PREFIX_INTERSECTION,
              getPrefixStatsIntersectionPrefixAttribute(field));
        } catch (EOFException e) {
          log.debug(e);
          doWrite = false;
        }
        // end loop over fields
      } while (doWrite);
      inTerm.close();
      closeables.remove(inTerm);
      inObject.close();
      closeables.remove(inObject);
      inObjectId.close();
      closeables.remove(inObjectId);
      inTmpDoc.close();
      closeables.remove(inTmpDoc);
      inTmpField.close();
      closeables.remove(inTmpField);

      // remove temporary files
      state.directory.deleteFile(mtasTmpDocFileName);
      state.directory.deleteFile(mtasTmpFieldFileName);
      // close indexDoc, indexObjectPosition and indexObjectParent
      CodecUtil.writeFooter(outDoc);
      outDoc.close();
      closeables.remove(outDoc);
      CodecUtil.writeFooter(outIndexObjectPosition);
      outIndexObjectPosition.close();
      closeables.remove(outIndexObjectPosition);
      CodecUtil.writeFooter(outIndexObjectParent);
      outIndexObjectParent.close();
      closeables.remove(outIndexObjectParent);
      CodecUtil.writeFooter(outIndexDocId);
      outIndexDocId.close();
      closeables.remove(outIndexDocId);
      CodecUtil.writeFooter(outField);
      outField.close();
      closeables.remove(outField);
    } catch (IOException e) {
      // ignore, can happen when merging segment already written by
      // delegateFieldsConsumer
      log.error(e);
    } finally {
      IOUtils.closeWhileHandlingException(closeables);
      try {
        state.directory.deleteFile(mtasTmpDocsFileName);
      } catch (IOException e) {
        log.debug(e);
      }
      try {
        state.directory.deleteFile(mtasTmpDocFileName);
      } catch (IOException e) {
        log.debug(e);
      }
      try {
        state.directory.deleteFile(mtasTmpFieldFileName);
      } catch (IOException e) {
        log.debug(e);
      }
    }
  }

  /**
   * Creates the object and register prefix.
   *
   * @param field
   *          the field
   * @param out
   *          the out
   * @param term
   *          the term
   * @param termRef
   *          the term ref
   * @param startPosition
   *          the start position
   * @param payload
   *          the payload
   * @param outPrefix
   *          the out prefix
   * @return the integer
   * @throws IOException
   *           Signals that an I/O exception has occurred.
   */
  private Integer createObjectAndRegisterPrefix(String field, IndexOutput out,
      BytesRef term, Long termRef, int startPosition, BytesRef payload,
      IndexOutput outPrefix) throws IOException {
    return createObjectAndRegisterPrefix(field, out, term, termRef,
        startPosition, payload, null, null, outPrefix);
  }

  /**
   * Creates the object and register prefix.
   *
   * @param field
   *          the field
   * @param out
   *          the out
   * @param term
   *          the term
   * @param termRef
   *          the term ref
   * @param startPosition
   *          the start position
   * @param payload
   *          the payload
   * @param startOffset
   *          the start offset
   * @param endOffset
   *          the end offset
   * @param outPrefix
   *          the out prefix
   * @return the integer
   * @throws IOException
   *           Signals that an I/O exception has occurred.
   */
  private Integer createObjectAndRegisterPrefix(String field, IndexOutput out,
      BytesRef term, Long termRef, int startPosition, BytesRef payload,
      Integer startOffset, Integer endOffset, IndexOutput outPrefix)
      throws IOException {
    try {
      Integer mtasId = null;
      String prefix = MtasToken.getPrefixFromValue(term.utf8ToString());
      if (payload != null) {
        MtasPayloadDecoder payloadDecoder = new MtasPayloadDecoder();
        payloadDecoder.init(startPosition, Arrays.copyOfRange(payload.bytes,
            payload.offset, (payload.offset + payload.length)));
        mtasId = payloadDecoder.getMtasId();
        Integer mtasParentId = payloadDecoder.getMtasParentId();
        byte[] mtasPayload = payloadDecoder.getMtasPayload();
        MtasPosition mtasPosition = payloadDecoder.getMtasPosition();
        MtasOffset mtasOffset = payloadDecoder.getMtasOffset();
        if (mtasOffset == null && startOffset != null) {
          mtasOffset = new MtasOffset(startOffset, endOffset);
        }
        MtasOffset mtasRealOffset = payloadDecoder.getMtasRealOffset();
        // only if really mtas object
        if (mtasId != null) {
          // compute flags
          int objectFlags = 0;
          if (mtasPosition != null) {
            if (mtasPosition.checkType(MtasPosition.POSITION_RANGE)) {
              objectFlags = objectFlags
                  | MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_RANGE;
              registerPrefixStatsRangePositionValue(field, prefix, outPrefix);
            } else if (mtasPosition.checkType(MtasPosition.POSITION_SET)) {
              objectFlags = objectFlags
                  | MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_SET;
              registerPrefixStatsSetPositionValue(field, prefix, outPrefix);
            } else {
              registerPrefixStatsSinglePositionValue(field, prefix, outPrefix);
            }
          } else {
            throw new IOException("no position");
          }
          if (mtasParentId != null) {
            objectFlags = objectFlags
                | MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PARENT;
          }
          if (mtasOffset != null) {
            objectFlags = objectFlags
                | MtasCodecPostingsFormat.MTAS_OBJECT_HAS_OFFSET;
          }
          if (mtasRealOffset != null) {
            objectFlags = objectFlags
                | MtasCodecPostingsFormat.MTAS_OBJECT_HAS_REALOFFSET;
          }
          if (mtasPayload != null) {
            objectFlags = objectFlags
                | MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PAYLOAD;
          }
          // create object
          out.writeVInt(mtasId);
          out.writeVInt(objectFlags);
          if ((objectFlags
              & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PARENT) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PARENT) {
            out.writeVInt(mtasParentId);
          }
          if ((objectFlags
              & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_RANGE) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_RANGE) {
            int tmpStart = mtasPosition.getStart();
            out.writeVInt(tmpStart);
            out.writeVInt((mtasPosition.getEnd() - tmpStart));
          } else if ((objectFlags
              & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_SET) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_SET) {
            int[] positions = mtasPosition.getPositions();
            out.writeVInt(positions.length);
            int tmpPrevious = 0;
            for (int position : positions) {
              out.writeVInt((position - tmpPrevious));
              tmpPrevious = position;
            }
          } else {
            out.writeVInt(mtasPosition.getStart());
          }
          if ((objectFlags
              & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_OFFSET) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_OFFSET) {
            int tmpStart = mtasOffset.getStart();
            out.writeVInt(mtasOffset.getStart());
            out.writeVInt((mtasOffset.getEnd() - tmpStart));
          }
          if ((objectFlags
              & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_REALOFFSET) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_REALOFFSET) {
            int tmpStart = mtasRealOffset.getStart();
            out.writeVInt(mtasRealOffset.getStart());
            out.writeVInt((mtasRealOffset.getEnd() - tmpStart));
          }
          if ((objectFlags
              & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PAYLOAD) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PAYLOAD) {
            if (mtasPayload != null) {
              out.writeVInt(mtasPayload.length);
              out.writeBytes(mtasPayload, mtasPayload.length);
            } else {
              out.writeVInt(0);
            }
          }
          out.writeVLong(termRef);
        } // storage token
      }
      return mtasId;
    } catch (Exception e) {
      log.error(e);
      throw new IOException(e);
    }
  }

  /**
   * Store tree.
   *
   * @param tree
   *          the tree
   * @param out
   *          the out
   * @param refApproxOffset
   *          the ref approx offset
   * @return the long
   * @throws IOException
   *           Signals that an I/O exception has occurred.
   */
  private Long storeTree(MtasTree<?> tree, IndexOutput out,
      long refApproxOffset) throws IOException {
    return storeTree(tree.close(), tree.isSinglePoint(),
        tree.isStorePrefixAndTermRef(), out, null, refApproxOffset);
  }

  /**
   * Store tree.
   *
   * @param node
   *          the node
   * @param isSinglePoint
   *          the is single point
   * @param storeAdditionalInformation
   *          the store additional information
   * @param out
   *          the out
   * @param nodeRefApproxOffset
   *          the node ref approx offset
   * @param refApproxOffset
   *          the ref approx offset
   * @return the long
   * @throws IOException
   *           Signals that an I/O exception has occurred.
   */
  private Long storeTree(MtasTreeNode<?> node, boolean isSinglePoint,
      boolean storeAdditionalInformation, IndexOutput out,
      Long nodeRefApproxOffset, long refApproxOffset) throws IOException {
    Long localNodeRefApproxOffset = nodeRefApproxOffset;
    if (node != null) {
      Boolean isRoot = false;
      if (localNodeRefApproxOffset == null) {
        localNodeRefApproxOffset = out.getFilePointer();
        isRoot = true;
      }
      Long fpIndexObjectPositionLeftChild;
      Long fpIndexObjectPositionRightChild;
      if (node.leftChild != null) {
        fpIndexObjectPositionLeftChild = storeTree(node.leftChild,
            isSinglePoint, storeAdditionalInformation, out,
            localNodeRefApproxOffset, refApproxOffset);
      } else {
        fpIndexObjectPositionLeftChild = (long) 0; // tmp
      }
      if (node.rightChild != null) {
        fpIndexObjectPositionRightChild = storeTree(node.rightChild,
            isSinglePoint, storeAdditionalInformation, out,
            localNodeRefApproxOffset, refApproxOffset);
      } else {
        fpIndexObjectPositionRightChild = (long) 0; // tmp
      }
      Long fpIndexObjectPosition = out.getFilePointer();
      if (node.leftChild == null) {
        fpIndexObjectPositionLeftChild = fpIndexObjectPosition;
      }
      if (node.rightChild == null) {
        fpIndexObjectPositionRightChild = fpIndexObjectPosition;
      }
      if (isRoot) {
        assert localNodeRefApproxOffset >= 0 : "nodeRefApproxOffset < 0 : "
            + localNodeRefApproxOffset;
        out.writeVLong(localNodeRefApproxOffset);
        byte flag = 0;
        if (isSinglePoint) {
          flag |= MtasTree.SINGLE_POSITION_TREE;
        }
        if (storeAdditionalInformation) {
          flag |= MtasTree.STORE_ADDITIONAL_ID;
        }
        out.writeByte(flag);
      }
      assert node.left >= 0 : "node.left < 0 : " + node.left;
      out.writeVInt(node.left);
      assert node.right >= 0 : "node.right < 0 : " + node.right;
      out.writeVInt(node.right);
      assert node.max >= 0 : "node.max < 0 : " + node.max;
      out.writeVInt(node.max);
      assert fpIndexObjectPositionLeftChild >= localNodeRefApproxOffset : "fpIndexObjectPositionLeftChild<nodeRefApproxOffset : "
          + fpIndexObjectPositionLeftChild + " and " + localNodeRefApproxOffset;
      out.writeVLong(
          (fpIndexObjectPositionLeftChild - localNodeRefApproxOffset));
      assert fpIndexObjectPositionRightChild >= localNodeRefApproxOffset : "fpIndexObjectPositionRightChild<nodeRefApproxOffset"
          + fpIndexObjectPositionRightChild + " and "
          + localNodeRefApproxOffset;
      out.writeVLong(
          (fpIndexObjectPositionRightChild - localNodeRefApproxOffset));
      if (!isSinglePoint) {
        out.writeVInt(node.ids.size());
      }
      HashMap<Integer, MtasTreeNodeId> ids = node.ids;
      Long objectRefCorrected;
      long objectRefCorrectedPrevious = 0;
      // sort refs
      List<MtasTreeNodeId> nodeIds = new ArrayList<>(ids.values());
      Collections.sort(nodeIds);
      if (isSinglePoint && (nodeIds.size() != 1)) {
        throw new IOException("singlePoint tree, but missing single point...");
      }
      int counter = 0;
      for (MtasTreeNodeId nodeId : nodeIds) {
        counter++;
        objectRefCorrected = (nodeId.ref - refApproxOffset);
        assert objectRefCorrected >= objectRefCorrectedPrevious : "objectRefCorrected<objectRefCorrectedPrevious : "
            + objectRefCorrected + " and " + objectRefCorrectedPrevious;
        out.writeVLong((objectRefCorrected - objectRefCorrectedPrevious));
        objectRefCorrectedPrevious = objectRefCorrected;
        if (storeAdditionalInformation) {
          assert nodeId.additionalId >= 0 : "nodeId.additionalId < 0 for item "
              + counter + " : " + nodeId.additionalId;
          out.writeVInt(nodeId.additionalId);
          assert nodeId.additionalRef >= 0 : "nodeId.additionalRef < 0 for item "
              + counter + " : " + nodeId.additionalRef;
          out.writeVLong(nodeId.additionalRef);
        }
      }
      return fpIndexObjectPosition;
    } else {
      return null;
    }
  }

  /**
   * Token stats add.
   *
   * @param min
   *          the min
   * @param max
   *          the max
   */
  private void tokenStatsAdd(int min, int max) {
    tokenStatsNumber++;
    if (tokenStatsMinPos == null) {
      tokenStatsMinPos = min;
    } else {
      tokenStatsMinPos = Math.min(tokenStatsMinPos, min);
    }
    if (tokenStatsMaxPos == null) {
      tokenStatsMaxPos = max;
    } else {
      tokenStatsMaxPos = Math.max(tokenStatsMaxPos, max);
    }
  }

  /**
   * Copy object and update stats.
   *
   * @param id
   *          the id
   * @param in
   *          the in
   * @param inRef
   *          the in ref
   * @param out
   *          the out
   * @throws IOException
   *           Signals that an I/O exception has occurred.
   */
  private void copyObjectAndUpdateStats(int id, IndexInput in, Long inRef,
      IndexOutput out) throws IOException {
    int mtasId;
    int objectFlags;
    // read
    in.seek(inRef);
    mtasId = in.readVInt();
    assert id == mtasId : "wrong id detected while copying object";
    objectFlags = in.readVInt();
    out.writeVInt(mtasId);
    out.writeVInt(objectFlags);
    if ((objectFlags
        & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PARENT) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PARENT) {
      out.writeVInt(in.readVInt());
    }
    if ((objectFlags
        & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_RANGE) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_RANGE) {
      int minPos = in.readVInt();
      int maxPos = in.readVInt();
      out.writeVInt(minPos);
      out.writeVInt(maxPos);
      tokenStatsAdd(minPos, maxPos);
    } else if ((objectFlags
        & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_SET) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_SET) {
      int size = in.readVInt();
      out.writeVInt(size);
      SortedSet<Integer> list = new TreeSet<>();
      int previousPosition = 0;
      for (int t = 0; t < size; t++) {
        int pos = in.readVInt();
        out.writeVInt(pos);
        previousPosition = (pos + previousPosition);
        list.add(previousPosition);
      }
      assert list.size() == size : "duplicate positions in set are not allowed";
      tokenStatsAdd(list.first(), list.last());
    } else {
      int pos = in.readVInt();
      out.writeVInt(pos);
      tokenStatsAdd(pos, pos);
    }
    if ((objectFlags
        & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_OFFSET) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_OFFSET) {
      out.writeVInt(in.readVInt());
      out.writeVInt(in.readVInt());
    }
    if ((objectFlags
        & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_REALOFFSET) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_REALOFFSET) {
      out.writeVInt(in.readVInt());
      out.writeVInt(in.readVInt());
    }
    if ((objectFlags
        & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PAYLOAD) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PAYLOAD) {
      int length = in.readVInt();
      out.writeVInt(length);
      byte[] payload = new byte[length];
      in.readBytes(payload, 0, length);
      out.writeBytes(payload, payload.length);
    }
    out.writeVLong(in.readVLong());
  }

  /*
   * (non-Javadoc)
   * 
   * @see org.apache.lucene.codecs.FieldsConsumer#close()
   */
  @Override
  public void close() throws IOException {
    delegateFieldsConsumer.close();
  }

}