Example usage for java.nio IntBuffer rewind

List of usage examples for java.nio IntBuffer rewind

  1. HOME
  2. Java
  3. java
  4. java.nio.*
  5. IntBuffer
  6. rewind

Introduction

In this page you can find the example usage for java.nio IntBuffer rewind.

Prototype

public final Buffer rewind()

Source Link

Document

Rewinds this buffer.

Usage

From source file:MainClass.java

public static void main(String[] args) {
    ByteBuffer bb = ByteBuffer.allocate(BSIZE);
    IntBuffer ib = bb.asIntBuffer();

    ib.put(new int[] { 1, 2, 7, 9, 3, 8, 6 });

    System.out.println(ib.get(3));
    ib.put(3, 1811);//w  w w  . j av a2 s .  co  m
    ib.rewind();
    while (ib.hasRemaining()) {
        int i = ib.get();
        if (i == 0)
            break; // Else we'll get the entire buffer
        System.out.println(i);
    }
}

From source file:MainClass.java

public static void main(String[] args) throws Exception {
    int port = 1919;

    SocketAddress address = new InetSocketAddress("127.0.0.1", port);
    SocketChannel client = SocketChannel.open(address);
    ByteBuffer buffer = ByteBuffer.allocate(4);
    IntBuffer view = buffer.asIntBuffer();

    for (int expected = 0;; expected++) {
        client.read(buffer);//from w ww.  j av  a  2 s . co m
        int actual = view.get();
        buffer.clear();
        view.rewind();

        if (actual != expected) {
            System.err.println("Expected " + expected + "; was " + actual);
            break;
        }
        System.out.println(actual);
    }
}

From source file:IntBufferDemo.java

public static void main(String[] args) {
    ByteBuffer bb = ByteBuffer.allocate(BSIZE);
    IntBuffer ib = bb.asIntBuffer();
    // Store an array of int:
    ib.put(new int[] { 11, 42, 47, 99, 143, 811, 1016 });
    // Absolute location read and write:
    System.out.println(ib.get(3));
    ib.put(3, 1811);/*ww w  . ja  v a 2  s  .c o  m*/
    ib.rewind();
    while (ib.hasRemaining()) {
        int i = ib.get();
        if (i == 0)
            break; // Else we'll get the entire buffer
        System.out.println(i);
    }

}

From source file:Main.java

public static int[] getIntArray(final IntBuffer buff) {
    if (buff == null) {
        return null;
    }//from  w w w.  j  a v a  2 s.  c  om
    buff.rewind();
    final int[] inds = new int[buff.limit()];
    for (int x = 0; x < inds.length; x++) {
        inds[x] = buff.get();
    }
    return inds;
}

From source file:Main.java

public static IntBuffer createIntBuffer(IntBuffer buf, final int size) {
    if (buf != null && buf.limit() == size) {
        buf.rewind();
        return buf;
    }/* w  w w. j  a v a 2s  .co  m*/

    buf = createIntBuffer(size);
    return buf;
}

From source file:Main.java

public static IntBuffer clone(final IntBuffer buf) {
    if (buf == null) {
        return null;
    }/* ww  w  .j  a v a2  s  .  c  om*/
    buf.rewind();

    final IntBuffer copy;
    if (buf.isDirect()) {
        copy = createIntBuffer(buf.limit());
    } else {
        copy = createIntBufferOnHeap(buf.limit());
    }
    copy.put(buf);

    return copy;
}

From source file:haven.Utils.java

public static IntBuffer bufcp(IntBuffer a) {
    a.rewind();
    IntBuffer ret = mkibuf(a.remaining());
    ret.put(a).rewind();//w  w w.ja v  a  2 s .  c  o  m
    return (ret);
}

From source file:com.metamx.druid.index.v1.IndexMerger.java

private static File makeIndexFiles(final List<IndexableAdapter> indexes, final File outDir,
        final ProgressIndicator progress, final List<String> mergedDimensions, final List<String> mergedMetrics,
        final Function<ArrayList<Iterable<Rowboat>>, Iterable<Rowboat>> rowMergerFn) throws IOException {
    Map<String, String> metricTypes = Maps.newTreeMap(Ordering.<String>natural().nullsFirst());
    for (IndexableAdapter adapter : indexes) {
        for (String metric : adapter.getAvailableMetrics()) {
            metricTypes.put(metric, adapter.getMetricType(metric));
        }/*  w w  w. j  a  v  a 2s.co  m*/
    }
    final Interval dataInterval;
    File v8OutDir = new File(outDir, "v8-tmp");
    v8OutDir.mkdirs();

    /*************  Main index.drd file **************/
    progress.progress();
    long startTime = System.currentTimeMillis();
    File indexFile = new File(v8OutDir, "index.drd");

    FileOutputStream fileOutputStream = null;
    FileChannel channel = null;
    try {
        fileOutputStream = new FileOutputStream(indexFile);
        channel = fileOutputStream.getChannel();
        channel.write(ByteBuffer.wrap(new byte[] { IndexIO.V8_VERSION }));

        GenericIndexed.fromIterable(mergedDimensions, GenericIndexed.stringStrategy).writeToChannel(channel);
        GenericIndexed.fromIterable(mergedMetrics, GenericIndexed.stringStrategy).writeToChannel(channel);

        DateTime minTime = new DateTime(Long.MAX_VALUE);
        DateTime maxTime = new DateTime(0l);

        for (IndexableAdapter index : indexes) {
            minTime = JodaUtils.minDateTime(minTime, index.getDataInterval().getStart());
            maxTime = JodaUtils.maxDateTime(maxTime, index.getDataInterval().getEnd());
        }

        dataInterval = new Interval(minTime, maxTime);
        serializerUtils.writeString(channel, String.format("%s/%s", minTime, maxTime));
    } finally {
        Closeables.closeQuietly(channel);
        channel = null;
        Closeables.closeQuietly(fileOutputStream);
        fileOutputStream = null;
    }
    IndexIO.checkFileSize(indexFile);
    log.info("outDir[%s] completed index.drd in %,d millis.", v8OutDir, System.currentTimeMillis() - startTime);

    /************* Setup Dim Conversions **************/
    progress.progress();
    startTime = System.currentTimeMillis();

    IOPeon ioPeon = new TmpFileIOPeon();
    ArrayList<FileOutputSupplier> dimOuts = Lists.newArrayListWithCapacity(mergedDimensions.size());
    Map<String, Integer> dimensionCardinalities = Maps.newHashMap();
    ArrayList<Map<String, IntBuffer>> dimConversions = Lists.newArrayListWithCapacity(indexes.size());

    for (IndexableAdapter index : indexes) {
        dimConversions.add(Maps.<String, IntBuffer>newHashMap());
    }

    for (String dimension : mergedDimensions) {
        final GenericIndexedWriter<String> writer = new GenericIndexedWriter<String>(ioPeon, dimension,
                GenericIndexed.stringStrategy);
        writer.open();

        List<Indexed<String>> dimValueLookups = Lists.newArrayListWithCapacity(indexes.size());
        DimValueConverter[] converters = new DimValueConverter[indexes.size()];
        for (int i = 0; i < indexes.size(); i++) {
            Indexed<String> dimValues = indexes.get(i).getDimValueLookup(dimension);
            if (dimValues != null) {
                dimValueLookups.add(dimValues);
                converters[i] = new DimValueConverter(dimValues);
            }
        }

        Iterable<String> dimensionValues = CombiningIterable.createSplatted(
                Iterables.transform(dimValueLookups, new Function<Indexed<String>, Iterable<String>>() {
                    @Override
                    public Iterable<String> apply(@Nullable Indexed<String> indexed) {
                        return Iterables.transform(indexed, new Function<String, String>() {
                            @Override
                            public String apply(@Nullable String input) {
                                return (input == null) ? "" : input;
                            }
                        });
                    }
                }), Ordering.<String>natural().nullsFirst());

        int count = 0;
        for (String value : dimensionValues) {
            value = value == null ? "" : value;
            writer.write(value);

            for (int i = 0; i < indexes.size(); i++) {
                DimValueConverter converter = converters[i];
                if (converter != null) {
                    converter.convert(value, count);
                }
            }

            ++count;
        }
        dimensionCardinalities.put(dimension, count);

        FileOutputSupplier dimOut = new FileOutputSupplier(IndexIO.makeDimFile(v8OutDir, dimension), true);
        dimOuts.add(dimOut);

        writer.close();
        serializerUtils.writeString(dimOut, dimension);
        ByteStreams.copy(writer.combineStreams(), dimOut);
        for (int i = 0; i < indexes.size(); ++i) {
            DimValueConverter converter = converters[i];
            if (converter != null) {
                dimConversions.get(i).put(dimension, converters[i].getConversionBuffer());
            }
        }

        ioPeon.cleanup();
    }
    log.info("outDir[%s] completed dim conversions in %,d millis.", v8OutDir,
            System.currentTimeMillis() - startTime);

    /************* Walk through data sets and merge them *************/
    progress.progress();
    startTime = System.currentTimeMillis();

    ArrayList<Iterable<Rowboat>> boats = Lists.newArrayListWithCapacity(indexes.size());

    for (int i = 0; i < indexes.size(); ++i) {
        final IndexableAdapter adapter = indexes.get(i);

        final int[] dimLookup = new int[mergedDimensions.size()];
        int count = 0;
        for (String dim : adapter.getAvailableDimensions()) {
            dimLookup[count] = mergedDimensions.indexOf(dim.toLowerCase());
            count++;
        }

        final int[] metricLookup = new int[mergedMetrics.size()];
        count = 0;
        for (String metric : adapter.getAvailableMetrics()) {
            metricLookup[count] = mergedMetrics.indexOf(metric);
            count++;
        }

        boats.add(new MMappedIndexRowIterable(
                Iterables.transform(indexes.get(i).getRows(), new Function<Rowboat, Rowboat>() {
                    @Override
                    public Rowboat apply(@Nullable Rowboat input) {
                        int[][] newDims = new int[mergedDimensions.size()][];
                        int j = 0;
                        for (int[] dim : input.getDims()) {
                            newDims[dimLookup[j]] = dim;
                            j++;
                        }

                        Object[] newMetrics = new Object[mergedMetrics.size()];
                        j = 0;
                        for (Object met : input.getMetrics()) {
                            newMetrics[metricLookup[j]] = met;
                            j++;
                        }

                        return new Rowboat(input.getTimestamp(), newDims, newMetrics, input.getRowNum(),
                                input.getDescriptions());
                    }
                }), mergedDimensions, dimConversions.get(i), i));
    }

    Iterable<Rowboat> theRows = rowMergerFn.apply(boats);

    CompressedLongsSupplierSerializer timeWriter = CompressedLongsSupplierSerializer.create(ioPeon,
            "little_end_time", IndexIO.BYTE_ORDER);

    timeWriter.open();

    ArrayList<VSizeIndexedWriter> forwardDimWriters = Lists.newArrayListWithCapacity(mergedDimensions.size());
    for (String dimension : mergedDimensions) {
        VSizeIndexedWriter writer = new VSizeIndexedWriter(ioPeon, dimension,
                dimensionCardinalities.get(dimension));
        writer.open();
        forwardDimWriters.add(writer);
    }

    ArrayList<MetricColumnSerializer> metWriters = Lists.newArrayListWithCapacity(mergedMetrics.size());
    for (Map.Entry<String, String> entry : metricTypes.entrySet()) {
        String metric = entry.getKey();
        String typeName = entry.getValue();
        if ("float".equals(typeName)) {
            metWriters.add(new FloatMetricColumnSerializer(metric, v8OutDir, ioPeon));
        } else {
            ComplexMetricSerde serde = ComplexMetrics.getSerdeForType(typeName);

            if (serde == null) {
                throw new ISE("Unknown type[%s]", typeName);
            }

            metWriters.add(new ComplexMetricColumnSerializer(metric, v8OutDir, ioPeon, serde));
        }
    }
    for (MetricColumnSerializer metWriter : metWriters) {
        metWriter.open();
    }

    int rowCount = 0;
    long time = System.currentTimeMillis();
    List<IntBuffer> rowNumConversions = Lists.newArrayListWithCapacity(indexes.size());
    for (IndexableAdapter index : indexes) {
        int[] arr = new int[index.getNumRows()];
        Arrays.fill(arr, INVALID_ROW);
        rowNumConversions.add(IntBuffer.wrap(arr));
    }

    final Map<String, String> descriptions = Maps.newHashMap();
    for (Rowboat theRow : theRows) {
        progress.progress();
        timeWriter.add(theRow.getTimestamp());

        final Object[] metrics = theRow.getMetrics();
        for (int i = 0; i < metrics.length; ++i) {
            metWriters.get(i).serialize(metrics[i]);
        }

        int[][] dims = theRow.getDims();
        for (int i = 0; i < dims.length; ++i) {
            List<Integer> listToWrite = (i >= dims.length || dims[i] == null) ? null : Ints.asList(dims[i]);
            forwardDimWriters.get(i).write(listToWrite);
        }

        for (Map.Entry<Integer, TreeSet<Integer>> comprisedRow : theRow.getComprisedRows().entrySet()) {
            final IntBuffer conversionBuffer = rowNumConversions.get(comprisedRow.getKey());

            for (Integer rowNum : comprisedRow.getValue()) {
                while (conversionBuffer.position() < rowNum) {
                    conversionBuffer.put(INVALID_ROW);
                }
                conversionBuffer.put(rowCount);
            }
        }

        if ((++rowCount % 500000) == 0) {
            log.info("outDir[%s] walked 500,000/%,d rows in %,d millis.", v8OutDir, rowCount,
                    System.currentTimeMillis() - time);
            time = System.currentTimeMillis();
        }

        descriptions.putAll(theRow.getDescriptions());
    }

    for (IntBuffer rowNumConversion : rowNumConversions) {
        rowNumConversion.rewind();
    }

    final File timeFile = IndexIO.makeTimeFile(v8OutDir, IndexIO.BYTE_ORDER);
    timeFile.delete();
    OutputSupplier<FileOutputStream> out = Files.newOutputStreamSupplier(timeFile, true);
    timeWriter.closeAndConsolidate(out);
    IndexIO.checkFileSize(timeFile);

    for (int i = 0; i < mergedDimensions.size(); ++i) {
        forwardDimWriters.get(i).close();
        ByteStreams.copy(forwardDimWriters.get(i).combineStreams(), dimOuts.get(i));
    }

    for (MetricColumnSerializer metWriter : metWriters) {
        metWriter.close();
    }

    ioPeon.cleanup();
    log.info("outDir[%s] completed walk through of %,d rows in %,d millis.", v8OutDir, rowCount,
            System.currentTimeMillis() - startTime);

    /************ Create Inverted Indexes *************/
    startTime = System.currentTimeMillis();

    final File invertedFile = new File(v8OutDir, "inverted.drd");
    Files.touch(invertedFile);
    out = Files.newOutputStreamSupplier(invertedFile, true);

    final File geoFile = new File(v8OutDir, "spatial.drd");
    Files.touch(geoFile);
    OutputSupplier<FileOutputStream> spatialOut = Files.newOutputStreamSupplier(geoFile, true);

    for (int i = 0; i < mergedDimensions.size(); ++i) {
        long dimStartTime = System.currentTimeMillis();
        String dimension = mergedDimensions.get(i);

        File dimOutFile = dimOuts.get(i).getFile();
        final MappedByteBuffer dimValsMapped = Files.map(dimOutFile);

        if (!dimension.equals(serializerUtils.readString(dimValsMapped))) {
            throw new ISE("dimensions[%s] didn't equate!?  This is a major WTF moment.", dimension);
        }
        Indexed<String> dimVals = GenericIndexed.read(dimValsMapped, GenericIndexed.stringStrategy);
        log.info("Starting dimension[%s] with cardinality[%,d]", dimension, dimVals.size());

        GenericIndexedWriter<ImmutableConciseSet> writer = new GenericIndexedWriter<ImmutableConciseSet>(ioPeon,
                dimension, ConciseCompressedIndexedInts.objectStrategy);
        writer.open();

        boolean isSpatialDim = "spatial".equals(descriptions.get(dimension));
        ByteBufferWriter<ImmutableRTree> spatialWriter = null;
        RTree tree = null;
        IOPeon spatialIoPeon = new TmpFileIOPeon();
        if (isSpatialDim) {
            spatialWriter = new ByteBufferWriter<ImmutableRTree>(spatialIoPeon, dimension,
                    IndexedRTree.objectStrategy);
            spatialWriter.open();
            tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50));
        }

        for (String dimVal : IndexedIterable.create(dimVals)) {
            progress.progress();
            List<Iterable<Integer>> convertedInverteds = Lists.newArrayListWithCapacity(indexes.size());
            for (int j = 0; j < indexes.size(); ++j) {
                convertedInverteds.add(new ConvertingIndexedInts(indexes.get(j).getInverteds(dimension, dimVal),
                        rowNumConversions.get(j)));
            }

            ConciseSet bitset = new ConciseSet();
            for (Integer row : CombiningIterable.createSplatted(convertedInverteds,
                    Ordering.<Integer>natural().nullsFirst())) {
                if (row != INVALID_ROW) {
                    bitset.add(row);
                }
            }

            writer.write(ImmutableConciseSet.newImmutableFromMutable(bitset));

            if (isSpatialDim && dimVal != null) {
                List<String> stringCoords = Lists.newArrayList(SPLITTER.split(dimVal));
                float[] coords = new float[stringCoords.size()];
                for (int j = 0; j < coords.length; j++) {
                    coords[j] = Float.valueOf(stringCoords.get(j));
                }
                tree.insert(coords, bitset);
            }
        }
        writer.close();

        serializerUtils.writeString(out, dimension);
        ByteStreams.copy(writer.combineStreams(), out);
        ioPeon.cleanup();

        log.info("Completed dimension[%s] in %,d millis.", dimension,
                System.currentTimeMillis() - dimStartTime);

        if (isSpatialDim) {
            spatialWriter.write(ImmutableRTree.newImmutableFromMutable(tree));
            spatialWriter.close();

            serializerUtils.writeString(spatialOut, dimension);
            ByteStreams.copy(spatialWriter.combineStreams(), spatialOut);
            spatialIoPeon.cleanup();
        }

    }

    log.info("outDir[%s] completed inverted.drd in %,d millis.", v8OutDir,
            System.currentTimeMillis() - startTime);

    final ArrayList<String> expectedFiles = Lists.newArrayList(Iterables.concat(
            Arrays.asList("index.drd", "inverted.drd", "spatial.drd",
                    String.format("time_%s.drd", IndexIO.BYTE_ORDER)),
            Iterables.transform(mergedDimensions, GuavaUtils.formatFunction("dim_%s.drd")),
            Iterables.transform(mergedMetrics,
                    GuavaUtils.formatFunction(String.format("met_%%s_%s.drd", IndexIO.BYTE_ORDER)))));

    Map<String, File> files = Maps.newLinkedHashMap();
    for (String fileName : expectedFiles) {
        files.put(fileName, new File(v8OutDir, fileName));
    }

    File smooshDir = new File(v8OutDir, "smoosher");
    smooshDir.mkdir();

    for (Map.Entry<String, File> entry : Smoosh.smoosh(v8OutDir, smooshDir, files).entrySet()) {
        entry.getValue().delete();
    }

    for (File file : smooshDir.listFiles()) {
        Files.move(file, new File(v8OutDir, file.getName()));
    }

    if (!smooshDir.delete()) {
        log.info("Unable to delete temporary dir[%s], contains[%s]", smooshDir,
                Arrays.asList(smooshDir.listFiles()));
        throw new IOException(String.format("Unable to delete temporary dir[%s]", smooshDir));
    }

    createIndexDrdFile(IndexIO.V8_VERSION, v8OutDir,
            GenericIndexed.fromIterable(mergedDimensions, GenericIndexed.stringStrategy),
            GenericIndexed.fromIterable(mergedMetrics, GenericIndexed.stringStrategy), dataInterval);

    IndexIO.DefaultIndexIOHandler.convertV8toV9(v8OutDir, outDir);
    FileUtils.deleteDirectory(v8OutDir);

    return outDir;
}

From source file:io.druid.segment.IndexMaker.java

private static int convertDims(final List<IndexableAdapter> adapters, final ProgressIndicator progress,
        final Iterable<Rowboat> theRows, final List<IntBuffer> rowNumConversions) throws IOException {
    final String section = "convert dims";
    progress.startSection(section);//from  w  w w.  j  a va 2 s .  c  om

    for (IndexableAdapter index : adapters) {
        int[] arr = new int[index.getNumRows()];
        Arrays.fill(arr, INVALID_ROW);
        rowNumConversions.add(IntBuffer.wrap(arr));
    }

    int rowCount = 0;
    for (Rowboat theRow : theRows) {
        for (Map.Entry<Integer, TreeSet<Integer>> comprisedRow : theRow.getComprisedRows().entrySet()) {
            final IntBuffer conversionBuffer = rowNumConversions.get(comprisedRow.getKey());

            for (Integer rowNum : comprisedRow.getValue()) {
                while (conversionBuffer.position() < rowNum) {
                    conversionBuffer.put(INVALID_ROW);
                }
                conversionBuffer.put(rowCount);
            }
        }

        if ((++rowCount % 500000) == 0) {
            progress.progressSection(section, String.format("Walked 500,000/%,d rows", rowCount));
        }
    }

    for (IntBuffer rowNumConversion : rowNumConversions) {
        rowNumConversion.rewind();
    }

    progress.stopSection(section);

    return rowCount;
}

From source file:io.druid.segment.IndexMerger.java

private static File makeIndexFiles(final List<IndexableAdapter> indexes, final File outDir,
        final ProgressIndicator progress, final List<String> mergedDimensions, final List<String> mergedMetrics,
        final Map<String, Object> segmentMetadata,
        final Function<ArrayList<Iterable<Rowboat>>, Iterable<Rowboat>> rowMergerFn, final IndexSpec indexSpec)
        throws IOException {
    final Map<String, ValueType> valueTypes = Maps.newTreeMap(Ordering.<String>natural().nullsFirst());
    final Map<String, String> metricTypeNames = Maps.newTreeMap(Ordering.<String>natural().nullsFirst());
    final Map<String, ColumnCapabilitiesImpl> columnCapabilities = Maps.newHashMap();

    for (IndexableAdapter adapter : indexes) {
        for (String dimension : adapter.getDimensionNames()) {
            ColumnCapabilitiesImpl mergedCapabilities = columnCapabilities.get(dimension);
            ColumnCapabilities capabilities = adapter.getCapabilities(dimension);
            if (mergedCapabilities == null) {
                mergedCapabilities = new ColumnCapabilitiesImpl();
                mergedCapabilities.setType(ValueType.STRING);
            }/*ww w  .j a va  2s  .c  o m*/
            columnCapabilities.put(dimension, mergedCapabilities.merge(capabilities));
        }
        for (String metric : adapter.getMetricNames()) {
            ColumnCapabilitiesImpl mergedCapabilities = columnCapabilities.get(metric);
            ColumnCapabilities capabilities = adapter.getCapabilities(metric);
            if (mergedCapabilities == null) {
                mergedCapabilities = new ColumnCapabilitiesImpl();
            }
            columnCapabilities.put(metric, mergedCapabilities.merge(capabilities));

            valueTypes.put(metric, capabilities.getType());
            metricTypeNames.put(metric, adapter.getMetricType(metric));
        }
    }

    final Interval dataInterval;
    File v8OutDir = new File(outDir, "v8-tmp");
    v8OutDir.mkdirs();

    /*************  Main index.drd file **************/
    progress.progress();
    long startTime = System.currentTimeMillis();
    File indexFile = new File(v8OutDir, "index.drd");

    try (FileOutputStream fileOutputStream = new FileOutputStream(indexFile);
            FileChannel channel = fileOutputStream.getChannel()) {
        channel.write(ByteBuffer.wrap(new byte[] { IndexIO.V8_VERSION }));

        GenericIndexed.fromIterable(mergedDimensions, GenericIndexed.STRING_STRATEGY).writeToChannel(channel);
        GenericIndexed.fromIterable(mergedMetrics, GenericIndexed.STRING_STRATEGY).writeToChannel(channel);

        DateTime minTime = new DateTime(JodaUtils.MAX_INSTANT);
        DateTime maxTime = new DateTime(JodaUtils.MIN_INSTANT);

        for (IndexableAdapter index : indexes) {
            minTime = JodaUtils.minDateTime(minTime, index.getDataInterval().getStart());
            maxTime = JodaUtils.maxDateTime(maxTime, index.getDataInterval().getEnd());
        }

        dataInterval = new Interval(minTime, maxTime);
        serializerUtils.writeString(channel, String.format("%s/%s", minTime, maxTime));
        serializerUtils.writeString(channel, mapper.writeValueAsString(indexSpec.getBitmapSerdeFactory()));
    }
    IndexIO.checkFileSize(indexFile);
    log.info("outDir[%s] completed index.drd in %,d millis.", v8OutDir, System.currentTimeMillis() - startTime);

    /************* Setup Dim Conversions **************/
    progress.progress();
    startTime = System.currentTimeMillis();

    IOPeon ioPeon = new TmpFileIOPeon();
    ArrayList<FileOutputSupplier> dimOuts = Lists.newArrayListWithCapacity(mergedDimensions.size());
    Map<String, Integer> dimensionCardinalities = Maps.newHashMap();
    ArrayList<Map<String, IntBuffer>> dimConversions = Lists.newArrayListWithCapacity(indexes.size());

    for (IndexableAdapter index : indexes) {
        dimConversions.add(Maps.<String, IntBuffer>newHashMap());
    }

    for (String dimension : mergedDimensions) {
        final GenericIndexedWriter<String> writer = new GenericIndexedWriter<String>(ioPeon, dimension,
                GenericIndexed.STRING_STRATEGY);
        writer.open();

        List<Indexed<String>> dimValueLookups = Lists.newArrayListWithCapacity(indexes.size());
        DimValueConverter[] converters = new DimValueConverter[indexes.size()];
        for (int i = 0; i < indexes.size(); i++) {
            Indexed<String> dimValues = indexes.get(i).getDimValueLookup(dimension);
            if (!isNullColumn(dimValues)) {
                dimValueLookups.add(dimValues);
                converters[i] = new DimValueConverter(dimValues);
            }
        }

        Iterable<String> dimensionValues = CombiningIterable.createSplatted(
                Iterables.transform(dimValueLookups, new Function<Indexed<String>, Iterable<String>>() {
                    @Override
                    public Iterable<String> apply(@Nullable Indexed<String> indexed) {
                        return Iterables.transform(indexed, new Function<String, String>() {
                            @Override
                            public String apply(@Nullable String input) {
                                return (input == null) ? "" : input;
                            }
                        });
                    }
                }), Ordering.<String>natural().nullsFirst());

        int count = 0;
        for (String value : dimensionValues) {
            value = value == null ? "" : value;
            writer.write(value);

            for (int i = 0; i < indexes.size(); i++) {
                DimValueConverter converter = converters[i];
                if (converter != null) {
                    converter.convert(value, count);
                }
            }

            ++count;
        }
        dimensionCardinalities.put(dimension, count);

        FileOutputSupplier dimOut = new FileOutputSupplier(IndexIO.makeDimFile(v8OutDir, dimension), true);
        dimOuts.add(dimOut);

        writer.close();
        serializerUtils.writeString(dimOut, dimension);
        ByteStreams.copy(writer.combineStreams(), dimOut);
        for (int i = 0; i < indexes.size(); ++i) {
            DimValueConverter converter = converters[i];
            if (converter != null) {
                dimConversions.get(i).put(dimension, converters[i].getConversionBuffer());
            }
        }

        ioPeon.cleanup();
    }
    log.info("outDir[%s] completed dim conversions in %,d millis.", v8OutDir,
            System.currentTimeMillis() - startTime);

    /************* Walk through data sets and merge them *************/
    progress.progress();
    startTime = System.currentTimeMillis();

    ArrayList<Iterable<Rowboat>> boats = Lists.newArrayListWithCapacity(indexes.size());

    for (int i = 0; i < indexes.size(); ++i) {
        final IndexableAdapter adapter = indexes.get(i);

        final int[] dimLookup = new int[mergedDimensions.size()];
        int count = 0;
        for (String dim : adapter.getDimensionNames()) {
            dimLookup[count] = mergedDimensions.indexOf(dim);
            count++;
        }

        final int[] metricLookup = new int[mergedMetrics.size()];
        count = 0;
        for (String metric : adapter.getMetricNames()) {
            metricLookup[count] = mergedMetrics.indexOf(metric);
            count++;
        }

        boats.add(new MMappedIndexRowIterable(
                Iterables.transform(indexes.get(i).getRows(), new Function<Rowboat, Rowboat>() {
                    @Override
                    public Rowboat apply(@Nullable Rowboat input) {
                        int[][] newDims = new int[mergedDimensions.size()][];
                        int j = 0;
                        for (int[] dim : input.getDims()) {
                            newDims[dimLookup[j]] = dim;
                            j++;
                        }

                        Object[] newMetrics = new Object[mergedMetrics.size()];
                        j = 0;
                        for (Object met : input.getMetrics()) {
                            newMetrics[metricLookup[j]] = met;
                            j++;
                        }

                        return new Rowboat(input.getTimestamp(), newDims, newMetrics, input.getRowNum());
                    }
                }), mergedDimensions, dimConversions.get(i), i));
    }

    Iterable<Rowboat> theRows = rowMergerFn.apply(boats);

    CompressedLongsSupplierSerializer timeWriter = CompressedLongsSupplierSerializer.create(ioPeon,
            "little_end_time", IndexIO.BYTE_ORDER, CompressedObjectStrategy.DEFAULT_COMPRESSION_STRATEGY);

    timeWriter.open();

    ArrayList<VSizeIndexedWriter> forwardDimWriters = Lists.newArrayListWithCapacity(mergedDimensions.size());
    for (String dimension : mergedDimensions) {
        VSizeIndexedWriter writer = new VSizeIndexedWriter(ioPeon, dimension,
                dimensionCardinalities.get(dimension));
        writer.open();
        forwardDimWriters.add(writer);
    }

    ArrayList<MetricColumnSerializer> metWriters = Lists.newArrayListWithCapacity(mergedMetrics.size());
    for (String metric : mergedMetrics) {
        ValueType type = valueTypes.get(metric);
        switch (type) {
        case LONG:
            metWriters.add(new LongMetricColumnSerializer(metric, v8OutDir, ioPeon));
            break;
        case FLOAT:
            metWriters.add(new FloatMetricColumnSerializer(metric, v8OutDir, ioPeon));
            break;
        case COMPLEX:
            final String typeName = metricTypeNames.get(metric);
            ComplexMetricSerde serde = ComplexMetrics.getSerdeForType(typeName);

            if (serde == null) {
                throw new ISE("Unknown type[%s]", typeName);
            }

            metWriters.add(new ComplexMetricColumnSerializer(metric, v8OutDir, ioPeon, serde));
            break;
        default:
            throw new ISE("Unknown type[%s]", type);
        }
    }

    for (MetricColumnSerializer metWriter : metWriters) {
        metWriter.open();
    }

    int rowCount = 0;
    long time = System.currentTimeMillis();
    List<IntBuffer> rowNumConversions = Lists.newArrayListWithCapacity(indexes.size());
    for (IndexableAdapter index : indexes) {
        int[] arr = new int[index.getNumRows()];
        Arrays.fill(arr, INVALID_ROW);
        rowNumConversions.add(IntBuffer.wrap(arr));
    }

    for (Rowboat theRow : theRows) {
        progress.progress();
        timeWriter.add(theRow.getTimestamp());

        final Object[] metrics = theRow.getMetrics();
        for (int i = 0; i < metrics.length; ++i) {
            metWriters.get(i).serialize(metrics[i]);
        }

        int[][] dims = theRow.getDims();
        for (int i = 0; i < dims.length; ++i) {
            List<Integer> listToWrite = (i >= dims.length || dims[i] == null) ? null : Ints.asList(dims[i]);
            forwardDimWriters.get(i).write(listToWrite);
        }

        for (Map.Entry<Integer, TreeSet<Integer>> comprisedRow : theRow.getComprisedRows().entrySet()) {
            final IntBuffer conversionBuffer = rowNumConversions.get(comprisedRow.getKey());

            for (Integer rowNum : comprisedRow.getValue()) {
                while (conversionBuffer.position() < rowNum) {
                    conversionBuffer.put(INVALID_ROW);
                }
                conversionBuffer.put(rowCount);
            }
        }

        if ((++rowCount % 500000) == 0) {
            log.info("outDir[%s] walked 500,000/%,d rows in %,d millis.", v8OutDir, rowCount,
                    System.currentTimeMillis() - time);
            time = System.currentTimeMillis();
        }
    }

    for (IntBuffer rowNumConversion : rowNumConversions) {
        rowNumConversion.rewind();
    }

    final File timeFile = IndexIO.makeTimeFile(v8OutDir, IndexIO.BYTE_ORDER);
    timeFile.delete();
    OutputSupplier<FileOutputStream> out = Files.newOutputStreamSupplier(timeFile, true);
    timeWriter.closeAndConsolidate(out);
    IndexIO.checkFileSize(timeFile);

    for (int i = 0; i < mergedDimensions.size(); ++i) {
        forwardDimWriters.get(i).close();
        ByteStreams.copy(forwardDimWriters.get(i).combineStreams(), dimOuts.get(i));
    }

    for (MetricColumnSerializer metWriter : metWriters) {
        metWriter.close();
    }

    ioPeon.cleanup();
    log.info("outDir[%s] completed walk through of %,d rows in %,d millis.", v8OutDir, rowCount,
            System.currentTimeMillis() - startTime);

    /************ Create Inverted Indexes *************/
    startTime = System.currentTimeMillis();

    final File invertedFile = new File(v8OutDir, "inverted.drd");
    Files.touch(invertedFile);
    out = Files.newOutputStreamSupplier(invertedFile, true);

    final File geoFile = new File(v8OutDir, "spatial.drd");
    Files.touch(geoFile);
    OutputSupplier<FileOutputStream> spatialOut = Files.newOutputStreamSupplier(geoFile, true);

    for (int i = 0; i < mergedDimensions.size(); ++i) {
        long dimStartTime = System.currentTimeMillis();
        String dimension = mergedDimensions.get(i);

        File dimOutFile = dimOuts.get(i).getFile();
        final MappedByteBuffer dimValsMapped = Files.map(dimOutFile);

        if (!dimension.equals(serializerUtils.readString(dimValsMapped))) {
            throw new ISE("dimensions[%s] didn't equate!?  This is a major WTF moment.", dimension);
        }
        Indexed<String> dimVals = GenericIndexed.read(dimValsMapped, GenericIndexed.STRING_STRATEGY);
        log.info("Starting dimension[%s] with cardinality[%,d]", dimension, dimVals.size());

        final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory();
        GenericIndexedWriter<ImmutableBitmap> writer = new GenericIndexedWriter<>(ioPeon, dimension,
                bitmapSerdeFactory.getObjectStrategy());
        writer.open();

        boolean isSpatialDim = columnCapabilities.get(dimension).hasSpatialIndexes();
        ByteBufferWriter<ImmutableRTree> spatialWriter = null;
        RTree tree = null;
        IOPeon spatialIoPeon = new TmpFileIOPeon();
        if (isSpatialDim) {
            BitmapFactory bitmapFactory = bitmapSerdeFactory.getBitmapFactory();
            spatialWriter = new ByteBufferWriter<ImmutableRTree>(spatialIoPeon, dimension,
                    new IndexedRTree.ImmutableRTreeObjectStrategy(bitmapFactory));
            spatialWriter.open();
            tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bitmapFactory), bitmapFactory);
        }

        for (String dimVal : IndexedIterable.create(dimVals)) {
            progress.progress();
            List<Iterable<Integer>> convertedInverteds = Lists.newArrayListWithCapacity(indexes.size());
            for (int j = 0; j < indexes.size(); ++j) {
                convertedInverteds.add(new ConvertingIndexedInts(
                        indexes.get(j).getBitmapIndex(dimension, dimVal), rowNumConversions.get(j)));
            }

            MutableBitmap bitset = bitmapSerdeFactory.getBitmapFactory().makeEmptyMutableBitmap();
            for (Integer row : CombiningIterable.createSplatted(convertedInverteds,
                    Ordering.<Integer>natural().nullsFirst())) {
                if (row != INVALID_ROW) {
                    bitset.add(row);
                }
            }

            writer.write(bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(bitset));

            if (isSpatialDim && dimVal != null) {
                List<String> stringCoords = Lists.newArrayList(SPLITTER.split(dimVal));
                float[] coords = new float[stringCoords.size()];
                for (int j = 0; j < coords.length; j++) {
                    coords[j] = Float.valueOf(stringCoords.get(j));
                }
                tree.insert(coords, bitset);
            }
        }
        writer.close();

        serializerUtils.writeString(out, dimension);
        ByteStreams.copy(writer.combineStreams(), out);
        ioPeon.cleanup();

        log.info("Completed dimension[%s] in %,d millis.", dimension,
                System.currentTimeMillis() - dimStartTime);

        if (isSpatialDim) {
            spatialWriter.write(ImmutableRTree.newImmutableFromMutable(tree));
            spatialWriter.close();

            serializerUtils.writeString(spatialOut, dimension);
            ByteStreams.copy(spatialWriter.combineStreams(), spatialOut);
            spatialIoPeon.cleanup();
        }

    }

    log.info("outDir[%s] completed inverted.drd in %,d millis.", v8OutDir,
            System.currentTimeMillis() - startTime);

    final ArrayList<String> expectedFiles = Lists.newArrayList(Iterables.concat(
            Arrays.asList("index.drd", "inverted.drd", "spatial.drd",
                    String.format("time_%s.drd", IndexIO.BYTE_ORDER)),
            Iterables.transform(mergedDimensions, GuavaUtils.formatFunction("dim_%s.drd")),
            Iterables.transform(mergedMetrics,
                    GuavaUtils.formatFunction(String.format("met_%%s_%s.drd", IndexIO.BYTE_ORDER)))));

    if (segmentMetadata != null && !segmentMetadata.isEmpty()) {
        writeMetadataToFile(new File(v8OutDir, "metadata.drd"), segmentMetadata);
        log.info("wrote metadata.drd in outDir[%s].", v8OutDir);

        expectedFiles.add("metadata.drd");
    }

    Map<String, File> files = Maps.newLinkedHashMap();
    for (String fileName : expectedFiles) {
        files.put(fileName, new File(v8OutDir, fileName));
    }

    File smooshDir = new File(v8OutDir, "smoosher");
    smooshDir.mkdir();

    for (Map.Entry<String, File> entry : Smoosh.smoosh(v8OutDir, smooshDir, files).entrySet()) {
        entry.getValue().delete();
    }

    for (File file : smooshDir.listFiles()) {
        Files.move(file, new File(v8OutDir, file.getName()));
    }

    if (!smooshDir.delete()) {
        log.info("Unable to delete temporary dir[%s], contains[%s]", smooshDir,
                Arrays.asList(smooshDir.listFiles()));
        throw new IOException(String.format("Unable to delete temporary dir[%s]", smooshDir));
    }

    createIndexDrdFile(IndexIO.V8_VERSION, v8OutDir,
            GenericIndexed.fromIterable(mergedDimensions, GenericIndexed.STRING_STRATEGY),
            GenericIndexed.fromIterable(mergedMetrics, GenericIndexed.STRING_STRATEGY), dataInterval,
            indexSpec.getBitmapSerdeFactory());

    IndexIO.DefaultIndexIOHandler.convertV8toV9(v8OutDir, outDir, indexSpec);
    FileUtils.deleteDirectory(v8OutDir);

    return outDir;
}