List of usage examples for javax.crypto.spec IvParameterSpec IvParameterSpec
public IvParameterSpec(byte[] iv)
iv
as the IV. From source file:org.opendedup.sdfs.filestore.cloud.BatchAwsS3ChunkStore.java
@Override public void writeHashBlobArchive(HashBlobArchive arc, long id) throws IOException { String haName = EncyptUtils.encHashArchiveName(id, Main.chunkStoreEncryptionEnabled); this.s3clientLock.readLock().lock(); try {// w w w . j a va 2 s . com int csz = toIntExact(arc.getFile().length()); ObjectMetadata md = new ObjectMetadata(); md.addUserMetadata("size", Integer.toString(arc.uncompressedLength.get())); md.addUserMetadata("lz4compress", Boolean.toString(Main.compress)); md.addUserMetadata("encrypt", Boolean.toString(Main.chunkStoreEncryptionEnabled)); md.addUserMetadata("compressedsize", Integer.toString(csz)); md.addUserMetadata("bsize", Integer.toString(arc.getLen())); md.addUserMetadata("objects", Integer.toString(arc.getSz())); md.addUserMetadata("bcompressedsize", Integer.toString(csz)); md.setContentType("binary/octet-stream"); md.setContentLength(csz); if (md5sum) { FileInputStream in = new FileInputStream(arc.getFile()); String mds = BaseEncoding.base64().encode(ServiceUtils.computeMD5Hash(in)); md.setContentMD5(mds); md.addUserMetadata("md5sum", mds); IOUtils.closeQuietly(in); } PutObjectRequest req = new PutObjectRequest(this.name, "blocks/" + haName, new FileInputStream(arc.getFile()), md); if (this.simpleS3) s3Service.putObject(req); else this.multiPartUpload(req); byte[] msg = Long.toString(System.currentTimeMillis()).getBytes(); String mds = BaseEncoding.base64().encode(ServiceUtils.computeMD5Hash(msg)); md.setContentMD5(mds); md.addUserMetadata("md5sum", mds); if (this.clustered) { md.setContentType("binary/octet-stream"); md.setContentLength(msg.length); PutObjectRequest creq = new PutObjectRequest(this.name, this.getClaimName(id), new ByteArrayInputStream(msg), md); s3Service.putObject(creq); } byte[] hs = arc.getHashesString().getBytes(); int sz = hs.length; if (Main.compress) { hs = CompressionUtils.compressLz4(hs); } byte[] ivb = PassPhrase.getByteIV(); if (Main.chunkStoreEncryptionEnabled) { hs = EncryptUtils.encryptCBC(hs, new IvParameterSpec(ivb)); } md = new ObjectMetadata(); md.addUserMetadata("size", Integer.toString(sz)); md.addUserMetadata("ivspec", BaseEncoding.base64().encode(ivb)); md.addUserMetadata("lastaccessed", "0"); md.addUserMetadata("lz4compress", Boolean.toString(Main.compress)); md.addUserMetadata("encrypt", Boolean.toString(Main.chunkStoreEncryptionEnabled)); md.addUserMetadata("compressedsize", Integer.toString(csz)); md.addUserMetadata("bsize", Integer.toString(arc.uncompressedLength.get())); md.addUserMetadata("bcompressedsize", Integer.toString(csz)); md.addUserMetadata("objects", Integer.toString(arc.getSz())); md.setContentType("binary/octet-stream"); md.setContentLength(hs.length); if (md5sum) { mds = BaseEncoding.base64().encode(ServiceUtils.computeMD5Hash(hs)); md.setContentMD5(mds); md.addUserMetadata("md5sum", mds); } req = new PutObjectRequest(this.name, "keys/" + haName, new ByteArrayInputStream(hs), md); s3Service.putObject(req); } catch (Throwable e) { SDFSLogger.getLog().fatal("unable to upload " + arc.getID() + " with id " + id, e); throw new IOException(e); } finally { this.s3clientLock.readLock().unlock(); } }
From source file:main.java.vasolsim.common.file.__NONUSEDREFERENCE_VaSOLSimExam.java
/** * Creates a cipher around a 128bit AES encryption initialized to decryption mode. * * @param key the key for the cipher//w w w .ja va 2 s. co m * * @return initialized cipher, null if there is an error. The popup manager is used to notify errors. */ protected static Cipher getDecryptionCipher(byte[] key) throws VaSolSimException { byte[] validatedKey = new byte[algorithmExpectedKeyLengthInBytes]; /* * Holy crap yes I know this is terribly insecure but if you give this function an encryption key smaller * than 128 bits or of a value than the one you have set in teh static fields we have bigger problems to * deal with. Also, its for school, who is actually going to read this but me haha. If you do, email me with * the subject line LAWL YOUR CRYPTO IS REALLY BAD and we can share a laugh; * * Anyways, if the key is too short, repeat the byte sequence (copy it) until the byte array key is long * enough. */ if (key.length < algorithmExpectedKeyLengthInBytes) { int invalidatedKeyIndex = 0; for (int index = 0; index < algorithmExpectedKeyLengthInBytes; index++) { validatedKey[index] = key[invalidatedKeyIndex++]; if (index >= key.length) invalidatedKeyIndex = 0; } } else if (key.length > algorithmExpectedKeyLengthInBytes) { if (!(key.length == 32 || key.length == 64)) throw new VaSolSimException( ERROR_MESSAGE_GENERIC_CRYPTO + "\n\nBAD KEY: HIGHER KEY NOT POT (256, 512)"); if (key.length == 32) { byte[] xorKey = new byte[16]; byte[] lowerHalf = new byte[16]; byte[] higherHalf = new byte[16]; System.arraycopy(key, 0, lowerHalf, 0, 16); System.arraycopy(key, 16, higherHalf, 0, 16); for (int index = 0; index < 16; index++) { int xorBytes = (int) lowerHalf[index] ^ (int) higherHalf[index]; xorKey[index] = (byte) (0xff & xorBytes); } validatedKey = xorKey; } else { byte[] lowerQuarterOne = new byte[16]; byte[] lowerQuarterTwo = new byte[16]; byte[] higherQuarterOne = new byte[16]; byte[] higherQuarterTwo = new byte[16]; byte[] lowerHalf = new byte[16]; byte[] higherHalf = new byte[16]; byte[] xorKey = new byte[16]; System.arraycopy(key, 0, lowerQuarterOne, 0, 16); System.arraycopy(key, 16, lowerQuarterTwo, 0, 16); System.arraycopy(key, 32, higherQuarterOne, 0, 16); System.arraycopy(key, 48, higherQuarterTwo, 0, 16); for (int index = 0; index < 16; index++) { int xorBytes = (int) lowerQuarterOne[index] ^ (int) lowerQuarterTwo[index]; lowerHalf[index] = (byte) (0xff & xorBytes); } for (int index = 0; index < 16; index++) { int xorBytes = (int) higherQuarterOne[index] ^ (int) higherQuarterTwo[index]; higherHalf[index] = (byte) (0xff & xorBytes); } for (int index = 0; index < 16; index++) { int xorBytes = (int) lowerHalf[index] ^ (int) higherHalf[index]; xorKey[index] = (byte) (0xff & xorBytes); } validatedKey = xorKey; } } else if (key.length == algorithmExpectedKeyLengthInBytes) validatedKey = key; Cipher cipher = null; try { System.out.println(validatedKey.length); cipher = Cipher.getInstance(serviceProviderInterface, serviceProvider); cipher.init(Cipher.DECRYPT_MODE, new SecretKeySpec(validatedKey, algorithm), new IvParameterSpec(IV)); } catch (NoSuchAlgorithmException e) { throw new VaSolSimException(ERROR_MESSAGE_GENERIC_CRYPTO + "\n\nBAD ALGORITHM\n" + e.toString() + "\n" + e.getCause() + "\n" + ExceptionUtils.getStackTrace(e), e); } catch (NoSuchProviderException e) { throw new VaSolSimException(ERROR_MESSAGE_GENERIC_CRYPTO + "\n\nBAD PROVIDER\n" + e.toString() + "\n" + e.getCause() + "\n" + ExceptionUtils.getStackTrace(e), e); } catch (NoSuchPaddingException e) { throw new VaSolSimException(ERROR_MESSAGE_GENERIC_CRYPTO + "\n\nNO SUCH PADDING\n" + e.toString() + "\n" + e.getCause() + "\n" + ExceptionUtils.getStackTrace(e), e); } catch (InvalidKeyException e) { throw new VaSolSimException(ERROR_MESSAGE_GENERIC_CRYPTO + "\n\nBAD KEY\n" + e.toString() + "\n" + e.getCause() + "\n" + ExceptionUtils.getStackTrace(e), e); } catch (InvalidAlgorithmParameterException e) { throw new VaSolSimException(ERROR_MESSAGE_GENERIC_CRYPTO + "\n\nBAD ALGORITHM PARAMS\n" + e.toString() + "\n" + e.getCause() + "\n" + ExceptionUtils.getStackTrace(e), e); } return cipher; }
From source file:main.java.vasolsim.common.file.__NONUSEDREFERENCE_VaSOLSimExam.java
/** * Creates a cipher around a statically defined DEFAULT_ENCRYPTION_ALGORITHM, AES 128bit by default, initialized to * encryption mode.// ww w . ja va 2 s . c om * * @param key secure hash of some sort of the key * * @return initialized cipher, null if internal exception * * @throws VaSolSimException any internal cryptographic related exceptions throw this for debugging to the user */ protected static Cipher getEncryptionCipher(byte[] key) throws VaSolSimException { byte[] validatedKey = new byte[algorithmExpectedKeyLengthInBytes]; /* * Holy crap yes I know this is terribly insecure but if you give this function an encryption key smaller * than 128 bits or of a value than the one you have set in teh static fields we have bigger problems to * deal with. Also, its for school, who is actually going to read this but me haha. If you do, email me with * the subject line LAWL YOUR CRYPTO IS REALLY BAD and we can share a laugh; * * Anyways, if the key is too short, repeat the byte sequence (copy it) until the byte array key is long * enough. */ if (key.length < algorithmExpectedKeyLengthInBytes) { int invalidatedKeyIndex = 0; for (int index = 0; index < algorithmExpectedKeyLengthInBytes; index++) { validatedKey[index] = key[invalidatedKeyIndex++]; if (index >= key.length) invalidatedKeyIndex = 0; } } else if (key.length > algorithmExpectedKeyLengthInBytes) { if (!(key.length == 32 || key.length == 64)) throw new VaSolSimException( ERROR_MESSAGE_GENERIC_CRYPTO + "\n\nBAD KEY: HIGHER KEY NOT POT (256, 512)"); if (key.length == 32) { byte[] xorKey = new byte[16]; byte[] lowerHalf = new byte[16]; byte[] higherHalf = new byte[16]; System.arraycopy(key, 0, lowerHalf, 0, 16); System.arraycopy(key, 16, higherHalf, 0, 16); for (int index = 0; index < 16; index++) { int xorBytes = (int) lowerHalf[index] ^ (int) higherHalf[index]; xorKey[index] = (byte) (0xff & xorBytes); } validatedKey = xorKey; } else { byte[] lowerQuarterOne = new byte[16]; byte[] lowerQuarterTwo = new byte[16]; byte[] higherQuarterOne = new byte[16]; byte[] higherQuarterTwo = new byte[16]; byte[] lowerHalf = new byte[16]; byte[] higherHalf = new byte[16]; byte[] xorKey = new byte[16]; System.arraycopy(key, 0, lowerQuarterOne, 0, 16); System.arraycopy(key, 16, lowerQuarterTwo, 0, 16); System.arraycopy(key, 32, higherQuarterOne, 0, 16); System.arraycopy(key, 48, higherQuarterTwo, 0, 16); for (int index = 0; index < 16; index++) { int xorBytes = (int) lowerQuarterOne[index] ^ (int) lowerQuarterTwo[index]; lowerHalf[index] = (byte) (0xff & xorBytes); } for (int index = 0; index < 16; index++) { int xorBytes = (int) higherQuarterOne[index] ^ (int) higherQuarterTwo[index]; higherHalf[index] = (byte) (0xff & xorBytes); } for (int index = 0; index < 16; index++) { int xorBytes = (int) lowerHalf[index] ^ (int) higherHalf[index]; xorKey[index] = (byte) (0xff & xorBytes); } validatedKey = xorKey; } } else if (key.length == algorithmExpectedKeyLengthInBytes) validatedKey = key; /* * Initialize the Cipher */ Cipher cipher = null; try { cipher = Cipher.getInstance(serviceProviderInterface, serviceProvider); cipher.init(Cipher.ENCRYPT_MODE, new SecretKeySpec(validatedKey, algorithm), new IvParameterSpec(IV)); } catch (NoSuchAlgorithmException e) { throw new VaSolSimException(ERROR_MESSAGE_GENERIC_CRYPTO + "\n\nBAD ALGORITHM\n" + e.toString() + "\n" + e.getCause() + "\n" + ExceptionUtils.getStackTrace(e), e); } catch (NoSuchProviderException e) { throw new VaSolSimException(ERROR_MESSAGE_GENERIC_CRYPTO + "\n\nBAD PROVIDER\n" + e.toString() + "\n" + e.getCause() + "\n" + ExceptionUtils.getStackTrace(e), e); } catch (NoSuchPaddingException e) { throw new VaSolSimException(ERROR_MESSAGE_GENERIC_CRYPTO + "\n\nNO SUCH PADDING\n" + e.toString() + "\n" + e.getCause() + "\n" + ExceptionUtils.getStackTrace(e), e); } catch (InvalidKeyException e) { throw new VaSolSimException(ERROR_MESSAGE_GENERIC_CRYPTO + "\n\nBAD KEY\n" + e.toString() + "\n" + e.getCause() + "\n" + ExceptionUtils.getStackTrace(e), e); } catch (InvalidAlgorithmParameterException e) { throw new VaSolSimException(ERROR_MESSAGE_GENERIC_CRYPTO + "\n\nBAD ALGORITHM PARAMS\n" + e.toString() + "\n" + e.getCause() + "\n" + ExceptionUtils.getStackTrace(e), e); } return cipher; }
From source file:com.datatorrent.lib.io.fs.AbstractFileOutputOperatorTest.java
private void checkCompressedFile(File file, List<Long> offsets, int startVal, int totalWindows, int totalRecords, SecretKey secretKey, byte[] iv) throws IOException { FileInputStream fis;/*from w ww .j a v a 2 s.co m*/ InputStream gss = null; GZIPInputStream gis = null; BufferedReader br = null; Cipher cipher = null; if (secretKey != null) { try { cipher = Cipher.getInstance("AES/CBC/PKCS5Padding"); IvParameterSpec ivps = new IvParameterSpec(iv); cipher.init(Cipher.DECRYPT_MODE, secretKey, ivps); } catch (Exception e) { throw new RuntimeException(e); } } int numWindows = 0; try { fis = new FileInputStream(file); //fis.skip(startOffset); gss = fis; if (secretKey != null) { try { /* Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding"); IvParameterSpec ivps = new IvParameterSpec(iv); cipher.init(Cipher.DECRYPT_MODE, secretKey, ivps); */ gss = new CipherInputStream(fis, cipher); } catch (Exception e) { throw new RuntimeException(e); } } long startOffset = 0; for (long offset : offsets) { // Skip initial case in case file is not yet created if (offset == 0) { continue; } long limit = offset - startOffset; LimitInputStream lis = new LimitInputStream(gss, limit); //gis = new GZIPInputStream(fis); gis = new GZIPInputStream(lis); br = new BufferedReader(new InputStreamReader(gis)); //br = new BufferedReader(new InputStreamReader(gss)); String eline = "" + (startVal + numWindows * 2); int count = 0; String line; while ((line = br.readLine()) != null) { Assert.assertEquals("File line", eline, line); ++count; if ((count % totalRecords) == 0) { ++numWindows; eline = "" + (startVal + numWindows * 2); } } startOffset = offset; } } catch (Exception e) { e.printStackTrace(); } finally { if (br != null) { br.close(); } else { if (gis != null) { gis.close(); } else if (gss != null) { gss.close(); } } } Assert.assertEquals("Total", totalWindows, numWindows); }
From source file:com.datatorrent.lib.io.fs.AbstractFileOutputOperatorTest.java
@Test public void testChainFilters() throws NoSuchAlgorithmException, IOException { EvenOddHDFSExactlyOnceWriter writer = new EvenOddHDFSExactlyOnceWriter(); KeyGenerator keygen = KeyGenerator.getInstance("AES"); keygen.init(128);//from w w w . j a v a2s . com final SecretKey secretKey = keygen.generateKey(); byte[] iv = "TestParam16bytes".getBytes(); final IvParameterSpec ivps = new IvParameterSpec(iv); FilterStreamProvider.FilterChainStreamProvider<FilterOutputStream, OutputStream> chainStreamProvider = new FilterStreamProvider.FilterChainStreamProvider<FilterOutputStream, OutputStream>(); chainStreamProvider.addStreamProvider(new FilterStreamCodec.GZipFilterStreamProvider()); // The filter is to keep track of the offsets to handle multi member gzip issue with openjdk // http://bugs.java.com/bugdatabase/view_bug.do?bug_id=4691425 final CounterFilterStreamContext evenCounterContext = new CounterFilterStreamContext(); final CounterFilterStreamContext oddCounterContext = new CounterFilterStreamContext(); chainStreamProvider.addStreamProvider( new FilterStreamProvider.SimpleFilterReusableStreamProvider<CounterFilterOutputStream, OutputStream>() { @Override protected FilterStreamContext<CounterFilterOutputStream> createFilterStreamContext( OutputStream outputStream) throws IOException { if (evenCounterContext.isDoInit()) { evenCounterContext.init(outputStream); return evenCounterContext; } else { oddCounterContext.init(outputStream); return oddCounterContext; } } }); chainStreamProvider.addStreamProvider( new FilterStreamProvider.SimpleFilterReusableStreamProvider<CipherOutputStream, OutputStream>() { @Override protected FilterStreamContext<CipherOutputStream> createFilterStreamContext( OutputStream outputStream) throws IOException { try { Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding"); cipher.init(Cipher.ENCRYPT_MODE, secretKey, ivps); return new FilterStreamCodec.CipherFilterStreamContext(outputStream, cipher); } catch (Exception e) { throw new IOException(e); } } }); writer.setFilterStreamProvider(chainStreamProvider); File evenFile = new File(testMeta.getDir(), EVEN_FILE); File oddFile = new File(testMeta.getDir(), ODD_FILE); List<Long> evenOffsets = new ArrayList<Long>(); List<Long> oddOffsets = new ArrayList<Long>(); writer.setFilePath(testMeta.getDir()); writer.setAlwaysWriteToTmp(false); writer.setup(testMeta.testOperatorContext); for (int i = 0; i < 10; ++i) { writer.beginWindow(i); for (int j = 0; j < 1000; ++j) { writer.input.put(i); } writer.endWindow(); if ((i % 2) == 1) { writer.beforeCheckpoint(i); evenOffsets.add(evenCounterContext.getCounter()); oddOffsets.add(oddCounterContext.getCounter()); } } writer.teardown(); /* evenOffsets.add(evenFile.length()); oddOffsets.add(oddFile.length()); */ checkCompressedFile(evenFile, evenOffsets, 0, 5, 1000, secretKey, iv); checkCompressedFile(oddFile, oddOffsets, 1, 5, 1000, secretKey, iv); }
From source file:com.tremolosecurity.provisioning.core.ProvisioningEngineImpl.java
@Override public Object decryptObject(EncryptedMessage msg) throws ProvisioningException { SecretKey key = this.cfgMgr .getSecretKey(this.cfgMgr.getCfg().getProvisioning().getQueueConfig().getEncryptionKeyName()); if (key == null) { throw new ProvisioningException("Queue message encryption key not found"); }/*from www.java2 s . c o m*/ IvParameterSpec spec = new IvParameterSpec(msg.getIv()); Cipher cipher; try { cipher = Cipher.getInstance("AES/CBC/PKCS5Padding"); cipher.init(Cipher.DECRYPT_MODE, key, spec); byte[] bytes = cipher.doFinal(msg.getMsg()); return JsonReader.jsonToJava(new String(bytes)); } catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | InvalidAlgorithmParameterException | IllegalBlockSizeException | BadPaddingException e) { throw new ProvisioningException("Could not decrypt message", e); } }
From source file:org.apache.xml.security.encryption.XMLCipher.java
/** * Decrypt an EncryptedData element to a byte array * * When passed in an EncryptedData node, returns the decryption * as a byte array./*w ww . jav a 2 s . co m*/ * * Does not modify the source document * @param element * @return the bytes resulting from the decryption * @throws XMLEncryptionException */ public byte[] decryptToByteArray(Element element) throws XMLEncryptionException { if (log.isDebugEnabled()) { log.debug("Decrypting to ByteArray..."); } if (cipherMode != DECRYPT_MODE) { log.error("XMLCipher unexpectedly not in DECRYPT_MODE..."); } EncryptedData encryptedData = factory.newEncryptedData(element); if (key == null) { KeyInfo ki = encryptedData.getKeyInfo(); if (ki != null) { try { // Add a EncryptedKey resolver ki.registerInternalKeyResolver( new EncryptedKeyResolver(encryptedData.getEncryptionMethod().getAlgorithm(), kek)); key = ki.getSecretKey(); } catch (KeyResolverException kre) { if (log.isDebugEnabled()) { log.debug(kre); } } } if (key == null) { log.error("XMLCipher::decryptElement called without a key and unable to resolve"); throw new XMLEncryptionException("encryption.nokey"); } } // Obtain the encrypted octets XMLCipherInput cipherInput = new XMLCipherInput(encryptedData); byte[] encryptedBytes = cipherInput.getBytes(); // Now create the working cipher String jceAlgorithm = JCEMapper.translateURItoJCEID(encryptedData.getEncryptionMethod().getAlgorithm()); if (log.isDebugEnabled()) { log.debug("JCE Algorithm = " + jceAlgorithm); } Cipher c; try { if (requestedJCEProvider == null) { c = Cipher.getInstance(jceAlgorithm); } else { c = Cipher.getInstance(jceAlgorithm, requestedJCEProvider); } } catch (NoSuchAlgorithmException nsae) { throw new XMLEncryptionException("empty", nsae); } catch (NoSuchProviderException nspre) { throw new XMLEncryptionException("empty", nspre); } catch (NoSuchPaddingException nspae) { throw new XMLEncryptionException("empty", nspae); } // Calculate the IV length and copy out // For now, we only work with Block ciphers, so this will work. // This should probably be put into the JCE mapper. int ivLen = c.getBlockSize(); byte[] ivBytes = new byte[ivLen]; // You may be able to pass the entire piece in to IvParameterSpec // and it will only take the first x bytes, but no way to be certain // that this will work for every JCE provider, so lets copy the // necessary bytes into a dedicated array. System.arraycopy(encryptedBytes, 0, ivBytes, 0, ivLen); IvParameterSpec iv = new IvParameterSpec(ivBytes); try { c.init(cipherMode, key, iv); } catch (InvalidKeyException ike) { throw new XMLEncryptionException("empty", ike); } catch (InvalidAlgorithmParameterException iape) { throw new XMLEncryptionException("empty", iape); } try { return c.doFinal(encryptedBytes, ivLen, encryptedBytes.length - ivLen); } catch (IllegalBlockSizeException ibse) { throw new XMLEncryptionException("empty", ibse); } catch (BadPaddingException bpe) { throw new XMLEncryptionException("empty", bpe); } }
From source file:org.opendedup.sdfs.filestore.cloud.BatchAwsS3ChunkStore.java
public StringResult getStringResult(String key) throws IOException, InterruptedException { this.s3clientLock.readLock().lock(); S3Object sobj = null;// ww w .j ava2 s . c om try { ObjectMetadata md = null; try { sobj = s3Service.getObject(getName(), key); md = s3Service.getObjectMetadata(this.name, key); } catch (Exception e) { throw new IOException(e); } int cl = (int) md.getContentLength(); byte[] data = new byte[cl]; DataInputStream in = null; try { in = new DataInputStream(sobj.getObjectContent()); in.readFully(data); } catch (Exception e) { throw new IOException(e); } finally { if (in != null) in.close(); } boolean encrypt = false; boolean compress = false; boolean lz4compress = false; Map<String, String> mp = this.getUserMetaData(md); byte[] ivb = null; if (mp.containsKey("ivspec")) { ivb = BaseEncoding.base64().decode(mp.get("ivspec")); } if (mp.containsKey("md5sum")) { try { byte[] shash = BaseEncoding.base64().decode(mp.get("md5sum")); byte[] chash = ServiceUtils.computeMD5Hash(data); if (!Arrays.equals(shash, chash)) throw new IOException("download corrupt at " + sobj.getKey()); } catch (NoSuchAlgorithmException e) { throw new IOException(e); } } int size = Integer.parseInt(mp.get("size")); encrypt = Boolean.parseBoolean(mp.get("encrypt")); lz4compress = Boolean.parseBoolean(mp.get("lz4compress")); boolean changed = false; Long hid = EncyptUtils.decHashArchiveName(sobj.getKey().substring(5), encrypt); if (this.clustered) mp = s3Service.getObjectMetadata(this.name, this.getClaimName(hid)).getUserMetadata(); if (mp.containsKey("deleted")) { mp.remove("deleted"); changed = true; } if (mp.containsKey("deleted-objects")) { mp.remove("deleted-objects"); changed = true; } if (encrypt) { if (ivb != null) { data = EncryptUtils.decryptCBC(data, new IvParameterSpec(ivb)); } else { data = EncryptUtils.decryptCBC(data); } } if (compress) data = CompressionUtils.decompressZLIB(data); else if (lz4compress) { data = CompressionUtils.decompressLz4(data, size); } String hast = new String(data); SDFSLogger.getLog().debug("reading hashes " + (String) mp.get("objects") + " from " + hid + " encn " + sobj.getKey().substring(5)); StringTokenizer ht = new StringTokenizer(hast, ","); StringResult st = new StringResult(); st.id = hid; st.st = ht; if (mp.containsKey("bsize")) { HashBlobArchive.currentLength.addAndGet(Integer.parseInt(mp.get("bsize"))); } if (mp.containsKey("bcompressedsize")) { HashBlobArchive.compressedLength.addAndGet(Integer.parseInt(mp.get("bcompressedsize"))); } if (changed) { try { md = sobj.getObjectMetadata(); md.setUserMetadata(mp); String kn = null; if (this.clustered) kn = this.getClaimName(hid); else kn = sobj.getKey(); this.updateObject(kn, md); } catch (Exception e) { throw new IOException(e); } } return st; } finally { if (sobj != null) sobj.close(); this.s3clientLock.readLock().unlock(); } }
From source file:org.opendedup.sdfs.filestore.cloud.BatchAwsS3ChunkStore.java
@Override public void uploadFile(File f, String to, String pp) throws IOException { this.s3clientLock.readLock().lock(); try {/*from w ww.jav a 2 s.com*/ InputStream in = null; while (to.startsWith(File.separator)) to = to.substring(1); String pth = pp + "/" + EncyptUtils.encString(to, Main.chunkStoreEncryptionEnabled); SDFSLogger.getLog().info("uploading " + f.getPath() + " to " + to + " pth " + pth); boolean isDir = false; boolean isSymlink = false; if (!OSValidator.isWindows()) { isDir = Files.readAttributes(f.toPath(), PosixFileAttributes.class, LinkOption.NOFOLLOW_LINKS) .isDirectory(); isSymlink = Files.readAttributes(f.toPath(), PosixFileAttributes.class, LinkOption.NOFOLLOW_LINKS) .isSymbolicLink(); } else { isDir = f.isDirectory(); } if (isSymlink) { try { HashMap<String, String> metaData = new HashMap<String, String>(); metaData.put("encrypt", Boolean.toString(Main.chunkStoreEncryptionEnabled)); metaData.put("lastmodified", Long.toString(f.lastModified())); String slp = EncyptUtils.encString(Files.readSymbolicLink(f.toPath()).toFile().getPath(), Main.chunkStoreEncryptionEnabled); metaData.put("symlink", slp); ObjectMetadata md = new ObjectMetadata(); md.setContentType("binary/octet-stream"); md.setContentLength(pth.getBytes().length); md.setUserMetadata(metaData); PutObjectRequest req = new PutObjectRequest(this.name, pth, new ByteArrayInputStream(pth.getBytes()), md); s3Service.putObject(req); if (this.isClustered()) this.checkoutFile(pth); } catch (Exception e1) { throw new IOException(e1); } } else if (isDir) { HashMap<String, String> metaData = FileUtils.getFileMetaData(f, Main.chunkStoreEncryptionEnabled); metaData.put("encrypt", Boolean.toString(Main.chunkStoreEncryptionEnabled)); metaData.put("lastmodified", Long.toString(f.lastModified())); metaData.put("directory", "true"); ObjectMetadata md = new ObjectMetadata(); md.setContentType("binary/octet-stream"); md.setContentLength(pth.getBytes().length); md.setUserMetadata(metaData); try { PutObjectRequest req = new PutObjectRequest(this.name, pth, new ByteArrayInputStream(pth.getBytes()), md); s3Service.putObject(req); if (this.isClustered()) this.checkoutFile(pth); } catch (Exception e1) { SDFSLogger.getLog().error("error uploading", e1); throw new IOException(e1); } } else { String rnd = RandomGUID.getGuid(); File p = new File(this.staged_sync_location, rnd); File z = new File(this.staged_sync_location, rnd + ".z"); File e = new File(this.staged_sync_location, rnd + ".e"); while (z.exists()) { rnd = RandomGUID.getGuid(); p = new File(this.staged_sync_location, rnd); z = new File(this.staged_sync_location, rnd + ".z"); e = new File(this.staged_sync_location, rnd + ".e"); } try { BufferedInputStream is = new BufferedInputStream(new FileInputStream(f)); BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(p)); IOUtils.copy(is, os); os.flush(); os.close(); is.close(); if (Main.compress) { CompressionUtils.compressFile(p, z); p.delete(); p = z; } byte[] ivb = null; if (Main.chunkStoreEncryptionEnabled) { try { ivb = PassPhrase.getByteIV(); EncryptUtils.encryptFile(p, e, new IvParameterSpec(ivb)); } catch (Exception e1) { throw new IOException(e1); } p.delete(); p = e; } String objName = pth; ObjectMetadata md = new ObjectMetadata(); Map<String, String> umd = FileUtils.getFileMetaData(f, Main.chunkStoreEncryptionEnabled); md.setUserMetadata(umd); md.addUserMetadata("lz4compress", Boolean.toString(Main.compress)); md.addUserMetadata("encrypt", Boolean.toString(Main.chunkStoreEncryptionEnabled)); if (ivb != null) md.addUserMetadata("ivspec", BaseEncoding.base64().encode(ivb)); md.addUserMetadata("lastmodified", Long.toString(f.lastModified())); if (simpleS3) { md.setContentType("binary/octet-stream"); in = new BufferedInputStream(new FileInputStream(p), 32768); try { if (md5sum) { byte[] md5Hash = ServiceUtils.computeMD5Hash(in); in.close(); String mds = BaseEncoding.base64().encode(md5Hash); md.setContentMD5(mds); md.addUserMetadata("md5sum", mds); } } catch (NoSuchAlgorithmException e2) { SDFSLogger.getLog().error("while hashing", e2); throw new IOException(e2); } in = new FileInputStream(p); md.setContentLength(p.length()); try { PutObjectRequest req = new PutObjectRequest(this.name, objName, in, md); s3Service.putObject(req); if (this.isClustered()) this.checkoutFile(pth); SDFSLogger.getLog().debug( "uploaded=" + f.getPath() + " lm=" + md.getUserMetadata().get("lastmodified")); } catch (AmazonS3Exception e1) { if (e1.getStatusCode() == 409) { try { s3Service.deleteObject(this.name, objName); this.uploadFile(f, to, pp); return; } catch (Exception e2) { throw new IOException(e2); } } else { throw new IOException(e1); } } catch (Exception e1) { // SDFSLogger.getLog().error("error uploading", e1); throw new IOException(e1); } } else { try { md.setContentType("binary/octet-stream"); in = new BufferedInputStream(new FileInputStream(p), 32768); byte[] md5Hash = ServiceUtils.computeMD5Hash(in); in.close(); String mds = BaseEncoding.base64().encode(md5Hash); md.setContentMD5(mds); md.addUserMetadata("md5sum", mds); in = new BufferedInputStream(new FileInputStream(p), 32768); md.setContentLength(p.length()); PutObjectRequest req = new PutObjectRequest(this.name, objName, in, md); multiPartUpload(req); if (this.isClustered()) this.checkoutFile(pth); } catch (AmazonS3Exception e1) { if (e1.getStatusCode() == 409) { try { s3Service.deleteObject(this.name, objName); this.uploadFile(f, to, pp); return; } catch (Exception e2) { throw new IOException(e2); } } else { throw new IOException(e1); } } catch (Exception e1) { // SDFSLogger.getLog().error("error uploading", e1); throw new IOException(e1); } } } finally { try { if (in != null) in.close(); } finally { p.delete(); z.delete(); e.delete(); } } } } finally { this.s3clientLock.readLock().unlock(); } }
From source file:org.opendedup.sdfs.filestore.cloud.BatchAwsS3ChunkStore.java
@Override public void downloadFile(String nm, File to, String pp) throws IOException { this.s3clientLock.readLock().lock(); try {//from w w w . j av a 2 s.c o m while (nm.startsWith(File.separator)) nm = nm.substring(1); String rnd = RandomGUID.getGuid(); File p = new File(this.staged_sync_location, rnd); File z = new File(this.staged_sync_location, rnd + ".uz"); File e = new File(this.staged_sync_location, rnd + ".de"); while (z.exists()) { rnd = RandomGUID.getGuid(); p = new File(this.staged_sync_location, rnd); z = new File(this.staged_sync_location, rnd + ".uz"); e = new File(this.staged_sync_location, rnd + ".de"); } if (nm.startsWith(File.separator)) nm = nm.substring(1); String haName = EncyptUtils.encString(nm, Main.chunkStoreEncryptionEnabled); Map<String, String> mp = null; byte[] shash = null; try { if (this.simpleS3) { S3Object obj = null; SDFSLogger.getLog().debug("downloading " + pp + "/" + haName); obj = s3Service.getObject(this.name, pp + "/" + haName); BufferedInputStream in = new BufferedInputStream(obj.getObjectContent()); BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(p)); IOUtils.copy(in, out); out.flush(); out.close(); in.close(); ObjectMetadata omd = s3Service.getObjectMetadata(name, pp + "/" + haName); mp = this.getUserMetaData(omd); SDFSLogger.getLog().debug("mp sz=" + mp.size()); try { if (obj != null) obj.close(); } catch (Exception e1) { } } else { SDFSLogger.getLog().debug("downloading " + pp + "/" + haName); this.multiPartDownload(pp + "/" + haName, p); ObjectMetadata omd = s3Service.getObjectMetadata(name, pp + "/" + haName); mp = this.getUserMetaData(omd); if (md5sum && mp.containsKey("md5sum")) { shash = BaseEncoding.base64().decode(omd.getUserMetaDataOf("md5sum")); } } if (shash != null && !FileUtils.fileValid(p, shash)) throw new IOException("file " + p.getPath() + " is corrupt"); boolean encrypt = false; boolean lz4compress = false; if (mp.containsKey("encrypt")) { encrypt = Boolean.parseBoolean(mp.get("encrypt")); } if (mp.containsKey("lz4compress")) { lz4compress = Boolean.parseBoolean(mp.get("lz4compress")); } byte[] ivb = null; if (mp.containsKey("ivspec")) { ivb = BaseEncoding.base64().decode(mp.get("ivspec")); } SDFSLogger.getLog().debug("compress=" + lz4compress + " " + mp.get("lz4compress")); if (mp.containsKey("symlink")) { if (OSValidator.isWindows()) throw new IOException("unable to restore symlinks to windows"); else { String spth = EncyptUtils.decString(mp.get("symlink"), encrypt); Path srcP = Paths.get(spth); Path dstP = Paths.get(to.getPath()); Files.createSymbolicLink(dstP, srcP); } } else if (mp.containsKey("directory")) { to.mkdirs(); FileUtils.setFileMetaData(to, mp, encrypt); p.delete(); } else { if (encrypt) { if (ivb != null) { EncryptUtils.decryptFile(p, e, new IvParameterSpec(ivb)); } else { EncryptUtils.decryptFile(p, e); } p.delete(); p = e; } if (lz4compress) { CompressionUtils.decompressFile(p, z); p.delete(); p = z; } File parent = to.getParentFile(); if (!parent.exists()) parent.mkdirs(); BufferedInputStream is = new BufferedInputStream(new FileInputStream(p)); BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(to)); IOUtils.copy(is, os); os.flush(); os.close(); is.close(); FileUtils.setFileMetaData(to, mp, encrypt); SDFSLogger.getLog().debug("updated " + to + " sz=" + to.length()); } } catch (Exception e1) { throw new IOException(e1); } finally { p.delete(); z.delete(); e.delete(); } } finally { this.s3clientLock.readLock().unlock(); } }