Example usage for org.apache.hadoop.conf Configuration set

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration set.

Prototype

public void set(String name, String value) 

Source Link

Document

Set the value of the name property.

Usage

From source file:co.cask.tephra.persist.HDFSTransactionStateStorageTest.java

License:Apache License

@BeforeClass
public static void setupBeforeClass() throws Exception {
    Configuration hConf = new Configuration();
    hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tmpFolder.newFolder().getAbsolutePath());

    dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
    conf = new Configuration(dfsCluster.getFileSystem().getConf());
}

---

From source file:co.cask.tephra.persist.LocalTransactionStateStorageTest.java

License:Apache License

@Override
protected Configuration getConfiguration(String testName) throws IOException {
    File testDir = tmpDir.newFolder(testName);
    Configuration conf = new Configuration();
    conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_LOCAL_DIR, testDir.getAbsolutePath());
    conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, SnapshotCodecV4.class.getName());
    return conf;/*from www .j  a  v  a2s  . c  o  m*/
}

---

From source file:co.cask.tephra.persist.LocalTransactionStateStorageTest.java

License:Apache License

@SuppressWarnings("deprecation")
@Test//w w  w . j  av a2s .c o m
public void testLongTxnBackwardsCompatibility() throws Exception {
    Configuration conf = getConfiguration("testLongTxnBackwardsCompatibility");

    // Use SnapshotCodec version 1
    String latestSnapshotCodec = conf.get(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES);
    conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName());

    TransactionStateStorage storage = null;
    try {
        storage = getStorage(conf);
        storage.startAndWait();

        // Create transaction snapshot and transaction edits with version when long running txns had -1 expiration.
        Collection<Long> invalid = Lists.newArrayList();
        NavigableMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap();
        long time1 = System.currentTimeMillis();
        long wp1 = time1 * TxConstants.MAX_TX_PER_MS;
        inProgress.put(wp1, new TransactionManager.InProgressTx(wp1 - 5, -1L));
        long time2 = time1 + 100;
        long wp2 = time2 * TxConstants.MAX_TX_PER_MS;
        inProgress.put(wp2, new TransactionManager.InProgressTx(wp2 - 50, time2 + 1000));
        Map<Long, Set<ChangeId>> committing = Maps.newHashMap();
        Map<Long, Set<ChangeId>> committed = Maps.newHashMap();
        TransactionSnapshot snapshot = new TransactionSnapshot(time2, 0, wp2, invalid, inProgress, committing,
                committed);
        long time3 = time1 + 200;
        long wp3 = time3 * TxConstants.MAX_TX_PER_MS;
        TransactionEdit edit1 = new TransactionEditV2(wp3, wp3 - 10, TransactionEdit.State.INPROGRESS, -1L,
                null, 0L, false, null);
        long time4 = time1 + 300;
        long wp4 = time4 * TxConstants.MAX_TX_PER_MS;
        TransactionEdit edit2 = new TransactionEditV2(wp4, wp4 - 10, TransactionEdit.State.INPROGRESS,
                time4 + 1000, null, 0L, false, null);

        // write snapshot and transaction edit
        storage.writeSnapshot(snapshot);
        TransactionLog log = storage.createLog(time2);
        log.append(edit1);
        log.append(edit2);
        log.close();

        // Start transaction manager
        conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, latestSnapshotCodec);
        long longTimeout = TimeUnit.SECONDS.toMillis(conf.getLong(TxConstants.Manager.CFG_TX_LONG_TIMEOUT,
                TxConstants.Manager.DEFAULT_TX_LONG_TIMEOUT));
        TransactionManager txm = new TransactionManager(conf, storage, new TxMetricsCollector());
        txm.startAndWait();
        try {
            // Verify that the txns in old format were read correctly.
            // There should be four in-progress transactions, and no invalid transactions
            TransactionSnapshot snapshot1 = txm.getCurrentState();
            Assert.assertEquals(ImmutableSortedSet.of(wp1, wp2, wp3, wp4), snapshot1.getInProgress().keySet());
            verifyInProgress(snapshot1.getInProgress().get(wp1), TransactionType.LONG, time1 + longTimeout);
            verifyInProgress(snapshot1.getInProgress().get(wp2), TransactionType.SHORT, time2 + 1000);
            verifyInProgress(snapshot1.getInProgress().get(wp3), TransactionType.LONG, time3 + longTimeout);
            verifyInProgress(snapshot1.getInProgress().get(wp4), TransactionType.SHORT, time4 + 1000);
            Assert.assertEquals(0, snapshot1.getInvalid().size());
        } finally {
            txm.stopAndWait();
        }
    } finally {
        if (storage != null) {
            storage.stopAndWait();
        }
    }
}

---

From source file:co.cask.tephra.snapshot.SnapshotCodecTest.java

License:Apache License

@Test
public void testMinimalDeserilization() throws Exception {
    long now = System.currentTimeMillis();
    long nowWritePointer = now * TxConstants.MAX_TX_PER_MS;
    /*//from  w  w  w .j  a  v  a 2  s  .  c o  m
     * Snapshot consisting of transactions at:
     */
    long tInvalid = nowWritePointer - 5; // t1 - invalid
    long readPtr = nowWritePointer - 4; // t2 - here and earlier committed
    long tLong = nowWritePointer - 3; // t3 - in-progress LONG
    long tCommitted = nowWritePointer - 2; // t4 - committed, changeset (r1, r2)
    long tShort = nowWritePointer - 1; // t5 - in-progress SHORT, canCommit called, changeset (r3, r4)

    TreeMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap(ImmutableSortedMap.of(tLong,
            new TransactionManager.InProgressTx(readPtr,
                    TransactionManager.getTxExpirationFromWritePointer(tLong,
                            TxConstants.Manager.DEFAULT_TX_LONG_TIMEOUT),
                    TransactionType.LONG),
            tShort, new TransactionManager.InProgressTx(readPtr, now + 1000, TransactionType.SHORT)));

    TransactionSnapshot snapshot = new TransactionSnapshot(now, readPtr, nowWritePointer,
            Lists.newArrayList(tInvalid), // invalid
            inProgress, ImmutableMap.<Long, Set<ChangeId>>of(tShort, Sets.<ChangeId>newHashSet()),
            ImmutableMap.<Long, Set<ChangeId>>of(tCommitted, Sets.<ChangeId>newHashSet()));

    Configuration conf1 = new Configuration();
    conf1.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, SnapshotCodecV4.class.getName());
    SnapshotCodecProvider provider1 = new SnapshotCodecProvider(conf1);

    byte[] byteArray;
    try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        provider1.encode(out, snapshot);
        byteArray = out.toByteArray();
    }

    // TransactionSnapshot and TransactionVisibilityState decode should pass now
    TransactionSnapshot txSnapshot = provider1.decode(new ByteArrayInputStream(byteArray));
    TransactionVisibilityState txVisibilityState = provider1
            .decodeTransactionVisibilityState(new ByteArrayInputStream(byteArray));
    assertTransactionVisibilityStateEquals(txSnapshot, txVisibilityState);

    // Corrupt the serialization byte array so that full deserialization will fail
    byteArray[byteArray.length - 1] = 'a';

    // TransactionVisibilityState decoding should pass since it doesn't decode the committing and committed changesets.
    TransactionVisibilityState txVisibilityState2 = provider1
            .decodeTransactionVisibilityState(new ByteArrayInputStream(byteArray));
    Assert.assertNotNull(txVisibilityState2);
    Assert.assertEquals(txVisibilityState, txVisibilityState2);
    Assert.assertEquals(readPtr, txVisibilityState2.getReadPointer());
    try {
        provider1.decode(new ByteArrayInputStream(byteArray));
        Assert.fail();
    } catch (RuntimeException e) {
        // expected since we modified the serialization bytes
    }
}

---

From source file:co.cask.tephra.snapshot.SnapshotCodecTest.java

License:Apache License

/**
 * In-progress LONG transactions written with DefaultSnapshotCodec will not have the type serialized as part of
 * the data.  Since these transactions also contain a non-negative expiration, we need to ensure we reset the type
 * correctly when the snapshot is loaded.
 *//*from   w  ww. j  ava  2s .c o  m*/
@Test
public void testDefaultToV3Compatibility() throws Exception {
    long now = System.currentTimeMillis();
    long nowWritePointer = now * TxConstants.MAX_TX_PER_MS;
    /*
     * Snapshot consisting of transactions at:
     */
    long tInvalid = nowWritePointer - 5; // t1 - invalid
    long readPtr = nowWritePointer - 4; // t2 - here and earlier committed
    long tLong = nowWritePointer - 3; // t3 - in-progress LONG
    long tCommitted = nowWritePointer - 2; // t4 - committed, changeset (r1, r2)
    long tShort = nowWritePointer - 1; // t5 - in-progress SHORT, canCommit called, changeset (r3, r4)

    TreeMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap(ImmutableSortedMap.of(tLong,
            new TransactionManager.InProgressTx(readPtr,
                    TransactionManager.getTxExpirationFromWritePointer(tLong,
                            TxConstants.Manager.DEFAULT_TX_LONG_TIMEOUT),
                    TransactionType.LONG),
            tShort, new TransactionManager.InProgressTx(readPtr, now + 1000, TransactionType.SHORT)));

    TransactionSnapshot snapshot = new TransactionSnapshot(now, readPtr, nowWritePointer,
            Lists.newArrayList(tInvalid), // invalid
            inProgress,
            ImmutableMap.<Long, Set<ChangeId>>of(tShort,
                    Sets.newHashSet(new ChangeId(new byte[] { 'r', '3' }),
                            new ChangeId(new byte[] { 'r', '4' }))),
            ImmutableMap.<Long, Set<ChangeId>>of(tCommitted, Sets
                    .newHashSet(new ChangeId(new byte[] { 'r', '1' }), new ChangeId(new byte[] { 'r', '2' }))));

    Configuration conf1 = new Configuration();
    conf1.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName());
    SnapshotCodecProvider provider1 = new SnapshotCodecProvider(conf1);

    ByteArrayOutputStream out = new ByteArrayOutputStream();
    try {
        provider1.encode(out, snapshot);
    } finally {
        out.close();
    }

    TransactionSnapshot snapshot2 = provider1.decode(new ByteArrayInputStream(out.toByteArray()));
    TransactionVisibilityState minTxSnapshot = provider1
            .decodeTransactionVisibilityState(new ByteArrayInputStream(out.toByteArray()));
    assertTransactionVisibilityStateEquals(snapshot2, minTxSnapshot);

    assertEquals(snapshot.getReadPointer(), snapshot2.getReadPointer());
    assertEquals(snapshot.getWritePointer(), snapshot2.getWritePointer());
    assertEquals(snapshot.getInvalid(), snapshot2.getInvalid());
    // in-progress transactions will have missing types
    assertNotEquals(snapshot.getInProgress(), snapshot2.getInProgress());
    assertEquals(snapshot.getCommittingChangeSets(), snapshot2.getCommittingChangeSets());
    assertEquals(snapshot.getCommittedChangeSets(), snapshot2.getCommittedChangeSets());

    // after fixing in-progress, full snapshot should match
    Map<Long, TransactionManager.InProgressTx> fixedInProgress = TransactionManager.txnBackwardsCompatCheck(
            TxConstants.Manager.DEFAULT_TX_LONG_TIMEOUT, 10000L, snapshot2.getInProgress());
    assertEquals(snapshot.getInProgress(), fixedInProgress);
    assertEquals(snapshot, snapshot2);
}

---

From source file:co.cask.tephra.snapshot.SnapshotCodecTest.java

License:Apache License

/**
 * Test full stack serialization for a TransactionManager migrating from DefaultSnapshotCodec to SnapshotCodecV3.
 *//*from   w  ww. j a  va2 s .  c  o  m*/
@Test
public void testDefaultToV3Migration() throws Exception {
    File testDir = tmpDir.newFolder("testDefaultToV3Migration");
    Configuration conf = new Configuration();
    conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName());
    conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_LOCAL_DIR, testDir.getAbsolutePath());

    Injector injector = Guice.createInjector(new ConfigModule(conf),
            new DiscoveryModules().getSingleNodeModules(), new TransactionModules().getSingleNodeModules());

    TransactionManager txManager = injector.getInstance(TransactionManager.class);
    txManager.startAndWait();

    txManager.startLong();

    // shutdown to force a snapshot
    txManager.stopAndWait();

    TransactionStateStorage txStorage = injector.getInstance(TransactionStateStorage.class);
    txStorage.startAndWait();

    // confirm that the in-progress entry is missing a type
    TransactionSnapshot snapshot = txStorage.getLatestSnapshot();
    TransactionVisibilityState txVisibilityState = txStorage.getLatestTransactionVisibilityState();
    assertTransactionVisibilityStateEquals(snapshot, txVisibilityState);
    assertNotNull(snapshot);
    assertEquals(1, snapshot.getInProgress().size());
    Map.Entry<Long, TransactionManager.InProgressTx> entry = snapshot.getInProgress().entrySet().iterator()
            .next();
    assertNull(entry.getValue().getType());
    txStorage.stopAndWait();

    // start a new Tx manager to test fixup
    Configuration conf2 = new Configuration();
    conf2.set(TxConstants.Manager.CFG_TX_SNAPSHOT_LOCAL_DIR, testDir.getAbsolutePath());
    conf2.setStrings(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName(),
            SnapshotCodecV3.class.getName());
    Injector injector2 = Guice.createInjector(new ConfigModule(conf2),
            new DiscoveryModules().getSingleNodeModules(), new TransactionModules().getSingleNodeModules());

    TransactionManager txManager2 = injector2.getInstance(TransactionManager.class);
    txManager2.startAndWait();

    // state should be recovered
    TransactionSnapshot snapshot2 = txManager2.getCurrentState();
    assertEquals(1, snapshot2.getInProgress().size());
    Map.Entry<Long, TransactionManager.InProgressTx> inProgressTx = snapshot2.getInProgress().entrySet()
            .iterator().next();
    assertEquals(TransactionType.LONG, inProgressTx.getValue().getType());

    // save a new snapshot
    txManager2.stopAndWait();

    TransactionStateStorage txStorage2 = injector2.getInstance(TransactionStateStorage.class);
    txStorage2.startAndWait();

    TransactionSnapshot snapshot3 = txStorage2.getLatestSnapshot();
    // full snapshot should have deserialized correctly without any fixups
    assertEquals(snapshot2.getInProgress(), snapshot3.getInProgress());
    assertEquals(snapshot2, snapshot3);
    txStorage2.stopAndWait();
}

---

From source file:co.cask.tephra.snapshot.SnapshotCodecTest.java

License:Apache License

@Test
public void testSnapshotCodecProviderConfiguration() throws Exception {
    Configuration conf = new Configuration(false);
    StringBuilder buf = new StringBuilder();
    for (Class c : TxConstants.Persist.DEFAULT_TX_SNAPHOT_CODEC_CLASSES) {
        if (buf.length() > 0) {
            buf.append(",\n    ");
        }/*from  w  ww .j  a va 2 s .  c  o m*/
        buf.append(c.getName());
    }
    conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, buf.toString());

    SnapshotCodecProvider codecProvider = new SnapshotCodecProvider(conf);
    SnapshotCodec v1codec = codecProvider.getCodecForVersion(new DefaultSnapshotCodec().getVersion());
    assertNotNull(v1codec);
    assertTrue(v1codec instanceof DefaultSnapshotCodec);

    SnapshotCodec v2codec = codecProvider.getCodecForVersion(new SnapshotCodecV2().getVersion());
    assertNotNull(v2codec);
    assertTrue(v2codec instanceof SnapshotCodecV2);

    SnapshotCodec v3codec = codecProvider.getCodecForVersion(new SnapshotCodecV3().getVersion());
    assertNotNull(v3codec);
    assertTrue(v3codec instanceof SnapshotCodecV3);

    SnapshotCodec v4codec = codecProvider.getCodecForVersion(new SnapshotCodecV4().getVersion());
    assertNotNull(v4codec);
    assertTrue(v4codec instanceof SnapshotCodecV4);
}

---

From source file:co.cask.tephra.snapshot.SnapshotCodecTest.java

License:Apache License

@Test
public void testSnapshotCodecV4() throws IOException, TransactionNotInProgressException {
    File testDir = tmpDir.newFolder("testSnapshotCodecV4");
    Configuration conf = new Configuration();
    conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, SnapshotCodecV4.class.getName());
    conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_LOCAL_DIR, testDir.getAbsolutePath());

    Injector injector = Guice.createInjector(new ConfigModule(conf),
            new DiscoveryModules().getSingleNodeModules(), new TransactionModules().getSingleNodeModules());

    TransactionManager txManager = injector.getInstance(TransactionManager.class);
    txManager.startAndWait();//w ww. jav a2s.com

    // Create a transaction and a checkpoint transaction
    Transaction transaction = txManager.startLong();
    Transaction checkpointTx = txManager.checkpoint(transaction);

    // shutdown to force a snapshot
    txManager.stopAndWait();

    // Validate the snapshot on disk
    TransactionStateStorage txStorage = injector.getInstance(TransactionStateStorage.class);
    txStorage.startAndWait();

    TransactionSnapshot snapshot = txStorage.getLatestSnapshot();
    TransactionVisibilityState txVisibilityState = txStorage.getLatestTransactionVisibilityState();
    assertTransactionVisibilityStateEquals(snapshot, txVisibilityState);

    Map<Long, TransactionManager.InProgressTx> inProgress = snapshot.getInProgress();
    Assert.assertEquals(1, inProgress.size());

    TransactionManager.InProgressTx inProgressTx = inProgress.get(transaction.getTransactionId());
    Assert.assertNotNull(inProgressTx);
    Assert.assertArrayEquals(checkpointTx.getCheckpointWritePointers(),
            inProgressTx.getCheckpointWritePointers().toLongArray());

    txStorage.stopAndWait();

    // start a new Tx manager to see if the transaction is restored correctly.
    Injector injector2 = Guice.createInjector(new ConfigModule(conf),
            new DiscoveryModules().getSingleNodeModules(), new TransactionModules().getSingleNodeModules());

    txManager = injector2.getInstance(TransactionManager.class);
    txManager.startAndWait();

    // state should be recovered
    snapshot = txManager.getCurrentState();
    inProgress = snapshot.getInProgress();
    Assert.assertEquals(1, inProgress.size());

    inProgressTx = inProgress.get(transaction.getTransactionId());
    Assert.assertNotNull(inProgressTx);
    Assert.assertArrayEquals(checkpointTx.getCheckpointWritePointers(),
            inProgressTx.getCheckpointWritePointers().toLongArray());

    // Should be able to commit the transaction
    Assert.assertTrue(txManager.canCommit(checkpointTx, Collections.<byte[]>emptyList()));
    Assert.assertTrue(txManager.commit(checkpointTx));

    // save a new snapshot
    txManager.stopAndWait();

    TransactionStateStorage txStorage2 = injector2.getInstance(TransactionStateStorage.class);
    txStorage2.startAndWait();

    snapshot = txStorage2.getLatestSnapshot();
    Assert.assertTrue(snapshot.getInProgress().isEmpty());
    txStorage2.stopAndWait();
}

---

From source file:co.cask.tephra.ThriftTransactionSystemTest.java

License:Apache License

@BeforeClass
public static void start() throws Exception {
    zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
    zkServer.startAndWait();/* ww w . j  a  v a2 s  . c  o  m*/

    Configuration conf = new Configuration();
    conf.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, false);
    conf.set(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM, zkServer.getConnectionStr());
    conf.set(TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY, "n-times");
    conf.setInt(TxConstants.Service.CFG_DATA_TX_CLIENT_ATTEMPTS, 1);

    Injector injector = Guice.createInjector(new ConfigModule(conf), new ZKModule(),
            new DiscoveryModules().getDistributedModules(),
            Modules.override(new TransactionModules().getDistributedModules()).with(new AbstractModule() {
                @Override
                protected void configure() {
                    bind(TransactionStateStorage.class).to(InMemoryTransactionStateStorage.class)
                            .in(Scopes.SINGLETON);
                }
            }), new TransactionClientModule());

    zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();

    // start a tx server
    txService = injector.getInstance(TransactionService.class);
    storage = injector.getInstance(TransactionStateStorage.class);
    txClient = injector.getInstance(TransactionSystemClient.class);
    try {
        LOG.info("Starting transaction service");
        txService.startAndWait();
    } catch (Exception e) {
        LOG.error("Failed to start service: ", e);
    }
}

---

From source file:co.cask.tephra.TransactionContextTest.java

License:Apache License

@BeforeClass
public static void setup() throws IOException {
    final Configuration conf = new Configuration();
    conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, SnapshotCodecV4.class.getName());
    conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, tmpFolder.newFolder().getAbsolutePath());
    Injector injector = Guice.createInjector(new ConfigModule(conf),
            new DiscoveryModules().getInMemoryModules(),
            Modules.override(new TransactionModules().getInMemoryModules()).with(new AbstractModule() {
                @Override//from ww w .j  av a2  s . c  o m
                protected void configure() {
                    TransactionManager txManager = new TransactionManager(conf);
                    txManager.startAndWait();
                    bind(TransactionManager.class).toInstance(txManager);
                    bind(TransactionSystemClient.class).to(DummyTxClient.class).in(Singleton.class);
                }
            }));

    txClient = (DummyTxClient) injector.getInstance(TransactionSystemClient.class);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »