Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.pinterest.pinlater.backends.redis; import com.pinterest.pinlater.thrift.PinLaterJobState; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.apache.commons.configuration.PropertiesConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; import java.io.IOException; import java.io.InputStream; import java.util.Set; /** * Encapsulates static utility methods used by the PinLaterRedisBackend and related classes. */ public final class RedisBackendUtils { private static final Logger LOG = LoggerFactory.getLogger(RedisBackendUtils.class); public static final String PINLATER_QUEUE_KEY_PREFIX = "plq"; public static final String PINLATER_JOB_ID_KEY_PREFIX = "plj"; public static final String PINLATER_QUEUE_NAMES_KEY_PREFIX = "pln"; public static final String PINLATER_JOB_HASH_KEY_PREFIX = "plh"; public static final String PINLATER_JOB_HASH_BODY_FIELD = "bd"; public static final String PINLATER_JOB_HASH_CUSTOM_STATUS_FIELD = "cs"; public static final String PINLATER_JOB_HASH_CREATED_AT_FIELD = "ca"; public static final String PINLATER_JOB_HASH_UPDATED_AT_FIELD = "ua"; public static final String PINLATER_JOB_HASH_ATTEMPTS_ALLOWED_FIELD = "al"; public static final String PINLATER_JOB_HASH_ATTEMPTS_REMAINING_FIELD = "ar"; public static final String PINLATER_JOB_HASH_CLAIM_DESCRIPTOR_FIELD = "cd"; // Default values to use when we cannot find from the job hash in redis. public static final String PINLATER_JOB_HASH_DEFAULT_BODY = ""; public static final String PINLATER_JOB_HASH_DEFAULT_CUSTOM_STATUS = ""; public static final long PINLATER_JOB_HASH_DEFAULT_CREATED_AT = 0L; public static final long PINLATER_JOB_HASH_DEFAULT_UPDATED_AT = 0L; public static final int PINLATER_JOB_HASH_DEFAULT_ATTEMPTS_ALLOWED = 0; public static final int PINLATER_JOB_HASH_DEFAULT_ATTEMPTS_REMAINING = 0; // Stats format used for incrementing when job hash not found. // The input should be (queueName, shardName, priority, operation). // e.g. "redis_job_hash_not_found_finishrepintasklowpri_1_1_dequeue". public static final String REDIS_JOB_HASH_NOT_FOUND_STATS_FORMAT = "redis_job_hash_not_found_%s_%s_%d_%s"; @VisibleForTesting public static final int CUSTOM_STATUS_SIZE_BYTES = 3000; /** * Characters that considered "magic characters" in LUA. */ public static final ImmutableSet<Character> LUA_MAGIC_CHARS = ImmutableSet.of('%', '^', '$', '(', ')', '.', '[', ']', '*', '+', '-', '?'); private RedisBackendUtils() { } /** * Constructs the redis queue sorted set key name given a queue name, shard id and priority. * * @param queueName Name of the queue. * @param shardName Redis shard name. * @param priority Priority level. * @param state Job state. * @return Redis sorted set key. */ public static String constructQueueRedisKey(String queueName, String shardName, int priority, PinLaterJobState state) { return String.format("%s_%s_%s.p%1d_s%1d", PINLATER_QUEUE_KEY_PREFIX, shardName, queueName, priority, state.getValue()); } /** * Constructs the redis hash key prefix for the given shard. * * The returned prefix is used to concatenate the jobId to form the full job hash key. This * function is used for LUA script, in which we concatenate the returned prefix with the jobId(s) * from redis. * * @param shardName Redis shard name. * @return Redis sorted set key. */ public static String constructHashRedisKeyPrefix(String queueName, String shardName) { return String.format("%s_%s_%s.", PINLATER_JOB_HASH_KEY_PREFIX, shardName, queueName); } /** * Constructs the redis hash key for the given job inside the given shard. * * @param shardName Redis shard name. * @param jobId Local job id inside the shard. * @return Redis hash key. */ public static String constructHashRedisKey(String queueName, String shardName, long jobId) { return String.format("%s%s", constructHashRedisKeyPrefix(queueName, shardName), jobId); } /** * Constructs the redis key used to increment and get the localId for jobs in the given queue. * * @param queueName Name of the queue. * @param shardName Redis shard name. * @return Redis string key. */ public static String constructJobIdRedisKey(String queueName, String shardName) { return String.format("%s_%s_%s", PINLATER_JOB_ID_KEY_PREFIX, shardName, queueName); } /** * Constructs the redis key used to store the queue names sorted list in redis. * * @param shardName Redis shard name. * @return Redis string key. */ public static String constructQueueNamesRedisKey(String shardName) { return String.format("%s_%s", PINLATER_QUEUE_NAMES_KEY_PREFIX, shardName); } /** * Creates the Redis shard map from config. * * @param redisConfigStream InputStream containing the Redis config json. * @param configuration PropertiesConfiguration object. * @return A map shardName -> RedisPools. */ public static ImmutableMap<String, RedisPools> buildShardMap(InputStream redisConfigStream, PropertiesConfiguration configuration) { RedisConfigSchema redisConfig; try { redisConfig = RedisConfigSchema.read(Preconditions.checkNotNull(redisConfigStream)); } catch (IOException e) { LOG.error("Failed to load redis configuration", e); throw new RuntimeException(e); } ImmutableMap.Builder<String, RedisPools> shardMapBuilder = new ImmutableMap.Builder<String, RedisPools>(); for (RedisConfigSchema.Shard shard : redisConfig.shards) { shardMapBuilder.put(shard.name, new RedisPools(configuration, shard.shardConfig.master.host, shard.shardConfig.master.port, shard.shardConfig.dequeueOnly)); } return shardMapBuilder.build(); } /** * Truncate the given custom status under CUSTOM_STATUS_SIZE_BYTES bytes. * * @param customStatus A string. * @return A string, as the truncated custom status. */ public static String truncateCustomStatus(String customStatus) { if (customStatus == null) { return customStatus; } return customStatus.length() > CUSTOM_STATUS_SIZE_BYTES ? customStatus.substring(0, CUSTOM_STATUS_SIZE_BYTES) : customStatus; } /** * Get all the queue names in the given shard. Note that it is the caller's responsibility to * close the connection after this method is called. * * @param conn Jedis connection for the given shard. * @param shardName The shard name for the given shard. * @return A set of strings, as all the queue names in that shard. */ public static Set<String> getQueueNames(Jedis conn, String shardName) { String queueNamesRedisKey = RedisBackendUtils.constructQueueNamesRedisKey(shardName); // TODO(jiacheng): Consider to do pagination if we get many queues on one shard. return conn.zrange(queueNamesRedisKey, 0, -1); } /** * Functions to parse the values from redis hash. If it is null, returns the default value. */ public static String parseJobHashBody(String s) { return s == null ? PINLATER_JOB_HASH_DEFAULT_BODY : s; } public static Integer parseJobHashAttemptsAllowed(String s) { return s == null ? PINLATER_JOB_HASH_DEFAULT_ATTEMPTS_ALLOWED : Integer.valueOf(s); } public static Integer parseJobHashAttemptsRemaining(String s) { return s == null ? PINLATER_JOB_HASH_DEFAULT_ATTEMPTS_REMAINING : Integer.valueOf(s); } public static String parseJobHashCustomStatus(String s) { return s == null ? PINLATER_JOB_HASH_DEFAULT_CUSTOM_STATUS : s; } public static Long parseJobHashCreatedAt(String s) { // Return milliseconds. return s == null ? PINLATER_JOB_HASH_DEFAULT_CREATED_AT : (long) (Double.valueOf(s) * 1000); } public static Long parseJobHashUpdatedAt(String s) { // Return milliseconds. return s == null ? PINLATER_JOB_HASH_DEFAULT_UPDATED_AT : (long) (Double.valueOf(s) * 1000); } /** * Returns a new string that escapes all LUA magic characters in the old string. * * @param oldStr Old string that we want to escape magic characters from. * @return New string with escaped magic characters. */ public static String escapeLuaMagicCharacters(String oldStr) { StringBuffer newStr = new StringBuffer(); for (int i = 0; i < oldStr.length(); i++) { char c = oldStr.charAt(i); // If this is a magic character, we want to escape it be prefixing it with a '%' char. if (LUA_MAGIC_CHARS.contains(c)) { newStr.append('%'); } newStr.append(c); } return newStr.toString(); } }