Java tutorial
/* * Copyright 2012-2013 Mathias Herberts * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.geoxp.oss.pig; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.PrintWriter; import java.io.StringReader; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.UUID; import org.apache.pig.EvalFunc; import org.apache.pig.data.DataByteArray; import org.apache.pig.data.Tuple; import org.apache.pig.impl.logicalLayer.schema.Schema; import org.apache.pig.impl.util.UDFContext; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.data.ACL; import org.bouncycastle.util.encoders.Base64; import com.geoxp.oss.CryptoHelper; import com.geoxp.oss.OSSException; import com.geoxp.oss.client.OSSClient; public class PigSecretStore extends EvalFunc<Object> { /** * Name of property containaing the name of the file storing the first halves of the secrets */ private static String PSS_FILE = "pss.file"; /** * Name of system property containing the OSS URL */ public static final String PSS_OSS_URL = "pss.oss.url"; /** * Fingerprint of SSH key to use for secret retrieval */ public static final String PSS_OSS_SSHKEY = "pss.oss.sshkey"; /** * ZooKeeper quorum */ public static final String PSS_ZK_QUORUM = "pss.zk.quorum"; /** * ZooKeeper root */ public static final String PSS_ZK_ROOT = "pss.zk.root"; /** * Secrets managed by PigSecretStore */ private static final Map<String, byte[]> secrets = new HashMap<String, byte[]>(); public PigSecretStore(String... args) { // // If PSS_FILE is null, attempt to read it from UDFContext // synchronized (secrets) { if (secrets.isEmpty()) { Properties props = UDFContext.getUDFContext().getUDFProperties(PigSecretStore.class); // // If props contains a key PSS_FILE, we are executing on the backend // if (props.containsKey(PSS_FILE)) { // // We're on the backend // // // Open PSS_File // try { // // Retrieve one half of the secrets // InputStream pssis = PigSecretStore.class.getClassLoader() .getResourceAsStream(props.getProperty(PSS_FILE)); BufferedReader br = new BufferedReader(new InputStreamReader(pssis)); // // Read zknode // String zknode = br.readLine(); // // Read all secret halves // while (true) { String line = br.readLine(); if (null == line) { break; } String[] tokens = line.split(" "); if (2 != tokens.length) { throw new RuntimeException("Invalid PSS_FILE content."); } secrets.put(tokens[0], Base64.decode(tokens[1].getBytes("UTF-8"))); } br.close(); // // Read ZooKeeper to retrieve second half of secrets // ZooKeeper zk = new ZooKeeper( UDFContext.getUDFContext().getClientSystemProps().getProperty(PSS_ZK_QUORUM), 5000, null); String zkdata = new String(zk.getData(zknode, false, null), "UTF-8"); zk.close(); br = new BufferedReader(new StringReader(zkdata)); // // Read all secret halves // while (true) { String line = br.readLine(); if (null == line) { break; } String[] tokens = line.split(" "); if (2 != tokens.length) { throw new RuntimeException("Invalid PSS_FILE content."); } byte[] otp = Base64.decode(tokens[1].getBytes("UTF-8")); // // Apply XOR // for (int i = 0; i < otp.length; i++) { secrets.get(tokens[0])[i] = (byte) (secrets.get(tokens[0])[i] ^ otp[i]); } } br.close(); } catch (InterruptedException ie) { throw new RuntimeException(ie); } catch (KeeperException ke) { throw new RuntimeException(ke); } catch (IOException ioe) { throw new RuntimeException(ioe); } } else { // // We're on the frontend // // // File is the first argument // props.setProperty(PSS_FILE, args[0]); List<String> secrets = Arrays.asList(args).subList(1, args.length); try { // // StringBuilder for ZK content // StringBuilder sb = new StringBuilder(); String uuid = UUID.randomUUID().toString(); String zknode = System.getProperty(PSS_ZK_ROOT) + "/" + uuid; // // Open PSS_FILE for writing // PrintWriter pw = new PrintWriter(args[0], "UTF-8"); // Write zknode first pw.println(zknode); // // Attempt to retrieve each listed secret from OSS and split them in two // halves using a OTP // for (String secretname : secrets) { byte[] secret = OSSClient.getSecret(System.getProperty(PSS_OSS_URL), secretname, System.getProperty(PSS_OSS_SSHKEY)); // // Generate OTP // byte[] otp = new byte[secret.length]; CryptoHelper.getSecureRandom().nextBytes(otp); // // Do an XOR between secret and OTP // for (int i = 0; i < secret.length; i++) { secret[i] = (byte) (secret[i] ^ otp[i]); } // // Output first half to file, second to sb for ZK // pw.print(secretname); pw.print(" "); pw.println(new String(Base64.encode(secret), "UTF-8")); sb.append(secretname); sb.append(" "); sb.append(new String(Base64.encode(otp), "UTF-8")); sb.append("\n"); } pw.close(); // // Write zookeeper content // ZooKeeper zk = new ZooKeeper(System.getProperty(PSS_ZK_QUORUM), 5000, null); List<ACL> acls = new ArrayList<ACL>(); acls.add(new ACL(ZooDefs.Perms.ALL, ZooDefs.Ids.ANYONE_ID_UNSAFE)); zk.create(zknode, sb.toString().getBytes("UTF-8"), acls, CreateMode.EPHEMERAL); } catch (InterruptedException ie) { throw new RuntimeException(ie); } catch (KeeperException ke) { throw new RuntimeException(ke); } catch (IOException ioe) { throw new RuntimeException(ioe); } catch (OSSException osse) { throw new RuntimeException(osse); } } } } } @Override public Object exec(Tuple input) throws IOException { // // If UDF is called with one parameter, it's simply a wrapper that does nothing. // Otherwise UDF performs unwrapping of the bytearray (2nd member of tuple) // using the secret whose name is the first member of the input tuple. // if (1 == input.size()) { return input.get(0); } else if (2 == input.size()) { return new DataByteArray(CryptoHelper.unwrapBlob(getSecret((String) input.get(0)), ((DataByteArray) input.get(1)).get())); } else { throw new IOException("Invalid input Tuple size, may be 1 or 2."); } } @Override public Schema outputSchema(Schema input) { return input; } public static byte[] getSecret(String name) { return secrets.get(name).clone(); } }