com.linkedin.pinot.controller.api.resources.PinotSegmentUploadRestletResource.java Source code

Java tutorial

Introduction

Here is the source code for com.linkedin.pinot.controller.api.resources.PinotSegmentUploadRestletResource.java

Source

/**
 * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.linkedin.pinot.controller.api.resources;

import com.google.common.base.Preconditions;
import com.linkedin.pinot.common.config.TableConfig;
import com.linkedin.pinot.common.config.TableNameBuilder;
import com.linkedin.pinot.common.metadata.ZKMetadataProvider;
import com.linkedin.pinot.common.metadata.segment.OfflineSegmentZKMetadata;
import com.linkedin.pinot.common.metadata.segment.SegmentZKMetadataCustomMapModifier;
import com.linkedin.pinot.common.metrics.ControllerMeter;
import com.linkedin.pinot.common.metrics.ControllerMetrics;
import com.linkedin.pinot.common.segment.SegmentMetadata;
import com.linkedin.pinot.common.segment.fetcher.SegmentFetcher;
import com.linkedin.pinot.common.segment.fetcher.SegmentFetcherFactory;
import com.linkedin.pinot.common.utils.CommonConstants;
import com.linkedin.pinot.common.utils.FileUploadDownloadClient;
import com.linkedin.pinot.common.utils.StringUtil;
import com.linkedin.pinot.common.utils.TarGzCompressionUtils;
import com.linkedin.pinot.common.utils.helix.HelixHelper;
import com.linkedin.pinot.common.utils.time.TimeUtils;
import com.linkedin.pinot.controller.ControllerConf;
import com.linkedin.pinot.controller.api.access.AccessControl;
import com.linkedin.pinot.controller.api.access.AccessControlFactory;
import com.linkedin.pinot.controller.helix.core.PinotHelixResourceManager;
import com.linkedin.pinot.controller.helix.core.PinotHelixSegmentOnlineOfflineStateModelGenerator;
import com.linkedin.pinot.controller.util.TableSizeReader;
import com.linkedin.pinot.controller.validation.StorageQuotaChecker;
import com.linkedin.pinot.core.segment.index.SegmentMetadataImpl;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.net.InetAddress;
import java.net.URISyntaxException;
import java.net.URLDecoder;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executor;
import javax.annotation.Nonnull;
import javax.inject.Inject;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.Encoded;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.container.AsyncResponse;
import javax.ws.rs.container.Suspended;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.apache.commons.httpclient.HttpConnectionManager;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.helix.ZNRecord;
import org.apache.helix.model.IdealState;
import org.glassfish.grizzly.http.server.Request;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import org.glassfish.jersey.server.ManagedAsync;
import org.joda.time.Interval;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONTokener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@Api(tags = Constants.SEGMENT_TAG)
@Path("/")
public class PinotSegmentUploadRestletResource {
    private static final Logger LOGGER = LoggerFactory.getLogger(PinotSegmentUploadRestletResource.class);

    @Inject
    PinotHelixResourceManager _pinotHelixResourceManager;

    @Inject
    ControllerConf _controllerConf;

    @Inject
    ControllerMetrics _controllerMetrics;

    @Inject
    HttpConnectionManager _connectionManager;

    @Inject
    Executor _executor;

    @Inject
    AccessControlFactory _accessControlFactory;

    @GET
    @Produces(MediaType.APPLICATION_JSON)
    @Path("/segments")
    @Deprecated
    public String listAllSegmentNames() throws Exception {
        FileUploadPathProvider provider = new FileUploadPathProvider(_controllerConf);
        final JSONArray ret = new JSONArray();
        for (final File file : provider.getBaseDataDir().listFiles()) {
            final String fileName = file.getName();
            if (fileName.equalsIgnoreCase("fileUploadTemp") || fileName.equalsIgnoreCase("schemasTemp")) {
                continue;
            }

            final String url = _controllerConf.generateVipUrl() + "/segments/" + fileName;
            ret.put(url);
        }
        return ret.toString();
    }

    @GET
    @Produces(MediaType.APPLICATION_JSON)
    @Path("/segments/{tableName}")
    @ApiOperation(value = "Lists names of all segments of a table", notes = "Lists names of all segment names of a table")
    public String listAllSegmentNames(
            @ApiParam(value = "Name of the table", required = true) @PathParam("tableName") String tableName,
            @ApiParam(value = "realtime|offline") @QueryParam("type") String tableTypeStr) throws Exception {
        JSONArray ret = new JSONArray();

        CommonConstants.Helix.TableType tableType = Constants.validateTableType(tableTypeStr);
        if (tableTypeStr == null) {
            ret.put(formatSegments(tableName, CommonConstants.Helix.TableType.OFFLINE));
            ret.put(formatSegments(tableName, CommonConstants.Helix.TableType.REALTIME));
        } else {
            ret.put(formatSegments(tableName, tableType));
        }
        return ret.toString();
    }

    @GET
    @Produces(MediaType.APPLICATION_OCTET_STREAM)
    @Path("/segments/{tableName}/{segmentName}")
    @ApiOperation(value = "Download a segment", notes = "Download a segment")
    public Response downloadSegment(
            @ApiParam(value = "Name of the table", required = true) @PathParam("tableName") String tableName,
            @ApiParam(value = "Name of the segment", required = true) @PathParam("segmentName") @Encoded String segmentName,
            @Context HttpHeaders httpHeaders) {
        // Validate data access
        boolean hasDataAccess;
        try {
            AccessControl accessControl = _accessControlFactory.create();
            hasDataAccess = accessControl.hasDataAccess(httpHeaders, tableName);
        } catch (Exception e) {
            throw new ControllerApplicationException(LOGGER,
                    "Caught exception while validating access to table: " + tableName,
                    Response.Status.INTERNAL_SERVER_ERROR, e);
        }
        if (!hasDataAccess) {
            throw new ControllerApplicationException(LOGGER, "No data access to table: " + tableName,
                    Response.Status.FORBIDDEN);
        }

        FileUploadPathProvider provider;
        try {
            provider = new FileUploadPathProvider(_controllerConf);
        } catch (Exception e) {
            throw new ControllerApplicationException(LOGGER, e.getMessage(), Response.Status.INTERNAL_SERVER_ERROR,
                    e);
        }
        try {
            segmentName = URLDecoder.decode(segmentName, "UTF-8");
        } catch (UnsupportedEncodingException e) {
            String errStr = "Could not decode segment name '" + segmentName + "'";
            throw new ControllerApplicationException(LOGGER, errStr, Response.Status.BAD_REQUEST);
        }
        final File dataFile = new File(provider.getBaseDataDir(), StringUtil.join("/", tableName, segmentName));
        if (!dataFile.exists()) {
            throw new ControllerApplicationException(LOGGER,
                    "Segment " + segmentName + " or table " + tableName + " not found", Response.Status.NOT_FOUND);
        }
        Response.ResponseBuilder builder = Response.ok(dataFile);
        builder.header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + dataFile.getName());
        builder.header(HttpHeaders.CONTENT_LENGTH, dataFile.length());
        return builder.build();
    }

    @DELETE
    @Produces(MediaType.APPLICATION_JSON)
    @Path("/segments/{tableName}/{segmentName}")
    @ApiOperation(value = "Deletes a segment", notes = "Deletes a segment")
    public SuccessResponse deleteOneSegment(
            @ApiParam(value = "Name of the table", required = true) @PathParam("tableName") String tableName,
            @ApiParam(value = "Name of the segment", required = true) @PathParam("segmentName") @Encoded String segmentName,
            @ApiParam(value = "realtime|offline", required = true) @QueryParam("type") String tableTypeStr) {
        CommonConstants.Helix.TableType tableType = Constants.validateTableType(tableTypeStr);
        if (tableType == null) {
            throw new ControllerApplicationException(LOGGER, "Table type must not be null",
                    Response.Status.BAD_REQUEST);
        }
        try {
            segmentName = URLDecoder.decode(segmentName, "UTF-8");
        } catch (UnsupportedEncodingException e) {
            String errStr = "Could not decode segment name '" + segmentName + "'";
            throw new ControllerApplicationException(LOGGER, errStr, Response.Status.BAD_REQUEST);
        }
        PinotSegmentRestletResource.toggleStateInternal(tableName, StateType.DROP, tableType, segmentName,
                _pinotHelixResourceManager);

        return new SuccessResponse("Segment deleted");
    }

    @DELETE
    @Produces(MediaType.APPLICATION_JSON)
    @Path("/segments/{tableName}")
    @ApiOperation(value = "Deletes all segments of a table", notes = "Deletes all segments of a table")
    public SuccessResponse deleteAllSegments(
            @ApiParam(value = "Name of the table", required = true) @PathParam("tableName") String tableName,
            @ApiParam(value = "realtime|offline", required = true) @QueryParam("type") String tableTypeStr) {
        CommonConstants.Helix.TableType tableType = Constants.validateTableType(tableTypeStr);
        if (tableType == null) {
            throw new ControllerApplicationException(LOGGER, "Table type must not be null",
                    Response.Status.BAD_REQUEST);
        }
        PinotSegmentRestletResource.toggleStateInternal(tableName, StateType.DROP, tableType, null,
                _pinotHelixResourceManager);

        return new SuccessResponse("All segments of table "
                + TableNameBuilder.forType(tableType).tableNameWithType(tableName) + " deleted");
    }

    @POST
    @ManagedAsync
    @Produces(MediaType.APPLICATION_JSON)
    @Consumes(MediaType.MULTIPART_FORM_DATA)
    @Path("/segments")
    @ApiOperation(value = "Upload a segment", notes = "Upload a segment as binary")
    public void uploadSegmentAsMultiPart(FormDataMultiPart multiPart,
            @ApiParam(value = "Whether to enable parallel push protection") @DefaultValue("false") @QueryParam(FileUploadDownloadClient.QueryParameters.ENABLE_PARALLEL_PUSH_PROTECTION) boolean enableParallelPushProtection,
            @Context HttpHeaders headers, @Context Request request, @Suspended final AsyncResponse asyncResponse) {
        try {
            asyncResponse
                    .resume(uploadSegmentInternal(multiPart, null, enableParallelPushProtection, headers, request));
        } catch (Throwable t) {
            asyncResponse.resume(t);
        }
    }

    @POST
    @ManagedAsync
    @Produces(MediaType.APPLICATION_JSON)
    @Consumes(MediaType.APPLICATION_JSON)
    @Path("/segments")
    @ApiOperation(value = "Upload a segment", notes = "Upload a segment as json")
    // TODO Does it even work if the segment is sent as a JSON body? Need to compare with the other API
    public void uploadSegmentAsJson(String segmentJsonStr, // If segment is present as json body
            @ApiParam(value = "Whether to enable parallel push protection") @DefaultValue("false") @QueryParam(FileUploadDownloadClient.QueryParameters.ENABLE_PARALLEL_PUSH_PROTECTION) boolean enableParallelPushProtection,
            @Context HttpHeaders headers, @Context Request request, @Suspended final AsyncResponse asyncResponse) {
        try {
            asyncResponse.resume(
                    uploadSegmentInternal(null, segmentJsonStr, enableParallelPushProtection, headers, request));
        } catch (Throwable t) {
            asyncResponse.resume(t);
        }
    }

    private SuccessResponse uploadSegmentInternal(FormDataMultiPart multiPart, String segmentJsonStr,
            boolean enableParallelPushProtection, HttpHeaders headers, Request request) {
        File tempTarredSegmentFile = null;
        File tempSegmentDir = null;

        if (headers != null) {
            // TODO: Add these headers into open source hadoop jobs
            LOGGER.info("HTTP Header {} is {}", CommonConstants.Controller.SEGMENT_NAME_HTTP_HEADER,
                    headers.getRequestHeader(CommonConstants.Controller.SEGMENT_NAME_HTTP_HEADER));
            LOGGER.info("HTTP Header {} is {}", CommonConstants.Controller.TABLE_NAME_HTTP_HEADER,
                    headers.getRequestHeader(CommonConstants.Controller.TABLE_NAME_HTTP_HEADER));
        }

        try {
            FileUploadPathProvider provider = new FileUploadPathProvider(_controllerConf);
            String tempSegmentName = "tmp-" + System.nanoTime();
            tempTarredSegmentFile = new File(provider.getFileUploadTmpDir(), tempSegmentName);
            tempSegmentDir = new File(provider.getTmpUntarredPath(), tempSegmentName);

            // Get upload type
            String uploadTypeStr = headers.getHeaderString(FileUploadDownloadClient.CustomHeaders.UPLOAD_TYPE);
            FileUploadDownloadClient.FileUploadType uploadType;
            if (uploadTypeStr != null) {
                uploadType = FileUploadDownloadClient.FileUploadType.valueOf(uploadTypeStr);
            } else {
                uploadType = FileUploadDownloadClient.FileUploadType.getDefaultUploadType();
            }

            String downloadURI = null;
            switch (uploadType) {
            case JSON:
            case URI:
                // Get download URI
                try {
                    downloadURI = getDownloadUri(uploadType, headers, segmentJsonStr);
                } catch (Exception e) {
                    throw new ControllerApplicationException(LOGGER, "Failed to get download URI",
                            Response.Status.BAD_REQUEST, e);
                }

                // Get segment fetcher based on the download URI
                SegmentFetcher segmentFetcher;
                try {
                    segmentFetcher = SegmentFetcherFactory.getInstance().getSegmentFetcherBasedOnURI(downloadURI);
                } catch (URISyntaxException e) {
                    throw new ControllerApplicationException(LOGGER,
                            "Caught exception while parsing download URI: " + downloadURI,
                            Response.Status.BAD_REQUEST, e);
                }
                if (segmentFetcher == null) {
                    throw new ControllerApplicationException(LOGGER,
                            "Failed to get segment fetcher for download URI: " + downloadURI,
                            Response.Status.BAD_REQUEST);
                }

                // Download segment tar file to local
                segmentFetcher.fetchSegmentToLocal(downloadURI, tempTarredSegmentFile);
                break;

            case TAR:
                try {
                    Map<String, List<FormDataBodyPart>> map = multiPart.getFields();
                    if (!validateMultiPart(map, null)) {
                        throw new ControllerApplicationException(LOGGER, "Invalid multi-part form",
                                Response.Status.BAD_REQUEST);
                    }
                    FormDataBodyPart bodyPart = map.values().iterator().next().get(0);
                    try (InputStream inputStream = bodyPart.getValueAs(InputStream.class);
                            OutputStream outputStream = new FileOutputStream(tempTarredSegmentFile)) {
                        IOUtils.copyLarge(inputStream, outputStream);
                    }
                } finally {
                    multiPart.cleanup();
                }
                break;

            default:
                throw new UnsupportedOperationException("Unsupported upload type: " + uploadType);
            }

            // While there is TarGzCompressionUtils.unTarOneFile, we use unTar here to unpack all files
            // in the segment in order to ensure the segment is not corrupted
            TarGzCompressionUtils.unTar(tempTarredSegmentFile, tempSegmentDir);
            File[] files = tempSegmentDir.listFiles();
            Preconditions.checkState(files != null && files.length == 1);
            File indexDir = files[0];

            SegmentMetadata segmentMetadata = new SegmentMetadataImpl(indexDir);
            String segmentName = segmentMetadata.getName();
            String offlineTableName = TableNameBuilder.OFFLINE.tableNameWithType(segmentMetadata.getTableName());
            String clientAddress = InetAddress.getByName(request.getRemoteAddr()).getHostName();
            LOGGER.info("Processing upload request for segment: {} of table: {} from client: {}", segmentName,
                    offlineTableName, clientAddress);
            uploadSegment(indexDir, segmentMetadata, tempTarredSegmentFile, downloadURI, provider,
                    enableParallelPushProtection, headers);

            return new SuccessResponse(
                    "Successfully uploaded segment: " + segmentName + " of table: " + offlineTableName);
        } catch (WebApplicationException e) {
            throw e;
        } catch (Exception e) {
            _controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_SEGMENT_UPLOAD_ERROR, 1L);
            throw new ControllerApplicationException(LOGGER,
                    "Caught internal server exception while uploading segment",
                    Response.Status.INTERNAL_SERVER_ERROR, e);
        } finally {
            FileUtils.deleteQuietly(tempTarredSegmentFile);
            FileUtils.deleteQuietly(tempSegmentDir);
        }
    }

    /**
     * Helper method to upload segment with the following steps:
     * <ul>
     *   <li>Check storage quota</li>
     *   <li>Check segment start/end time</li>
     *   <li>For new segment (non-refresh), directly add the segment</li>
     *   <li>
     *     For REFRESH case
     *     <ul>
     *       <li>Check IF-MATCH CRC if existing</li>
     *       <li>Lock the segment if parallel push protection enabled</li>
     *       <li>Update the custom map in segment ZK metadata</li>
     *       <li>Update the segment ZK metadata (this will also unlock the segment)</li>
     *     </ul>
     *   </li>
     * </ul>
     */
    private void uploadSegment(File indexDir, SegmentMetadata segmentMetadata, File tempTarredSegmentFile,
            String downloadUrl, FileUploadPathProvider provider, boolean enableParallelPushProtection,
            HttpHeaders headers) throws IOException, JSONException {
        String rawTableName = segmentMetadata.getTableName();
        String offlineTableName = TableNameBuilder.OFFLINE.tableNameWithType(rawTableName);
        String segmentName = segmentMetadata.getName();
        TableConfig offlineTableConfig = ZKMetadataProvider
                .getOfflineTableConfig(_pinotHelixResourceManager.getPropertyStore(), offlineTableName);

        if (offlineTableConfig == null) {
            throw new ControllerApplicationException(LOGGER,
                    "Failed to find table config for table: " + offlineTableName, Response.Status.NOT_FOUND);
        }

        // Check quota
        StorageQuotaChecker.QuotaCheckerResponse quotaResponse = checkStorageQuota(indexDir, segmentMetadata,
                offlineTableConfig);
        if (!quotaResponse.isSegmentWithinQuota) {
            throw new ControllerApplicationException(LOGGER, "Quota check failed for segment: " + segmentName
                    + " of table: " + offlineTableName + ", reason: " + quotaResponse.reason,
                    Response.Status.FORBIDDEN);
        }

        // Check time range
        if (!isSegmentTimeValid(segmentMetadata)) {
            throw new ControllerApplicationException(LOGGER,
                    "Invalid segment start/end time for segment: " + segmentName + " of table: " + offlineTableName,
                    Response.Status.NOT_ACCEPTABLE);
        }

        ZNRecord znRecord = _pinotHelixResourceManager.getSegmentMetadataZnRecord(offlineTableName, segmentName);

        // Brand new segment, not refresh, directly add the segment
        if (znRecord == null) {
            if (downloadUrl == null) {
                downloadUrl = moveSegmentToPermanentDirectory(provider, rawTableName, segmentName,
                        tempTarredSegmentFile);
            }
            _pinotHelixResourceManager.addNewSegment(segmentMetadata, downloadUrl);
            return;
        }

        // Segment already exists, refresh if necessary
        OfflineSegmentZKMetadata existingSegmentZKMetadata = new OfflineSegmentZKMetadata(znRecord);
        long existingCrc = existingSegmentZKMetadata.getCrc();

        // Check if CRC match when IF-MATCH header is set
        String expectedCrcStr = headers.getHeaderString(HttpHeaders.IF_MATCH);
        if (expectedCrcStr != null) {
            long expectedCrc;
            try {
                expectedCrc = Long.parseLong(expectedCrcStr);
            } catch (NumberFormatException e) {
                throw new ControllerApplicationException(LOGGER,
                        "Caught exception for segment: " + segmentName + " of table: " + offlineTableName
                                + " while parsing IF-MATCH CRC: \"" + expectedCrcStr + "\"",
                        Response.Status.PRECONDITION_FAILED);
            }
            if (expectedCrc != existingCrc) {
                throw new ControllerApplicationException(LOGGER,
                        "For segment: " + segmentName + " of table: " + offlineTableName + ", expected CRC: "
                                + expectedCrc + " does not match existing CRC: " + existingCrc,
                        Response.Status.PRECONDITION_FAILED);
            }
        }

        // Check segment upload start time when parallel push protection enabled
        if (enableParallelPushProtection) {
            // When segment upload start time is larger than 0, that means another upload is in progress
            long segmentUploadStartTime = existingSegmentZKMetadata.getSegmentUploadStartTime();
            if (segmentUploadStartTime > 0) {
                if (System.currentTimeMillis() - segmentUploadStartTime > _controllerConf
                        .getSegmentUploadTimeoutInMillis()) {
                    // Last segment upload does not finish properly, replace the segment
                    LOGGER.error("Segment: {} of table: {} was not properly uploaded, replacing it", segmentName,
                            offlineTableName);
                    _controllerMetrics.addMeteredGlobalValue(ControllerMeter.NUMBER_SEGMENT_UPLOAD_TIMEOUT_EXCEEDED,
                            1L);
                } else {
                    // Another segment upload is in progress
                    throw new ControllerApplicationException(LOGGER,
                            "Another segment upload is in progress for segment: " + segmentName + " of table: "
                                    + offlineTableName + ", retry later",
                            Response.Status.CONFLICT);
                }
            }

            // Lock the segment by setting the upload start time in ZK
            existingSegmentZKMetadata.setSegmentUploadStartTime(System.currentTimeMillis());
            if (!_pinotHelixResourceManager.updateZkMetadata(existingSegmentZKMetadata, znRecord.getVersion())) {
                throw new ControllerApplicationException(LOGGER, "Failed to lock the segment: " + segmentName
                        + " of table: " + offlineTableName + ", retry later", Response.Status.CONFLICT);
            }
        }

        // Reset segment upload start time to unlock the segment later
        // NOTE: reset this value even if parallel push protection is not enabled so that segment can recover in case
        // previous segment upload did not finish properly and the parallel push protection is turned off
        existingSegmentZKMetadata.setSegmentUploadStartTime(-1);

        try {
            // Modify the custom map in segment ZK metadata
            String segmentZKMetadataCustomMapModifierStr = headers.getHeaderString(
                    FileUploadDownloadClient.CustomHeaders.SEGMENT_ZK_METADATA_CUSTOM_MAP_MODIFIER);
            SegmentZKMetadataCustomMapModifier segmentZKMetadataCustomMapModifier;
            if (segmentZKMetadataCustomMapModifierStr != null) {
                segmentZKMetadataCustomMapModifier = new SegmentZKMetadataCustomMapModifier(
                        segmentZKMetadataCustomMapModifierStr);
            } else {
                // By default, use REPLACE modify mode
                segmentZKMetadataCustomMapModifier = new SegmentZKMetadataCustomMapModifier(
                        SegmentZKMetadataCustomMapModifier.ModifyMode.REPLACE, null);
            }
            existingSegmentZKMetadata.setCustomMap(
                    segmentZKMetadataCustomMapModifier.modifyMap(existingSegmentZKMetadata.getCustomMap()));

            // Update ZK metadata and refresh the segment if necessary
            long newCrc = Long.valueOf(segmentMetadata.getCrc());
            if (newCrc == existingCrc) {
                // New segment is the same as the existing one, only update ZK metadata without refresh the segment
                if (!_pinotHelixResourceManager.updateZkMetadata(existingSegmentZKMetadata)) {
                    throw new RuntimeException("Failed to update ZK metadata for segment: " + segmentName
                            + " of table: " + offlineTableName);
                }
            } else {
                // New segment is different with the existing one, update ZK metadata and refresh the segment
                if (downloadUrl == null) {
                    downloadUrl = moveSegmentToPermanentDirectory(provider, rawTableName, segmentName,
                            tempTarredSegmentFile);
                }
                _pinotHelixResourceManager.refreshSegment(segmentMetadata, existingSegmentZKMetadata, downloadUrl);
            }
        } catch (Exception e) {
            if (!_pinotHelixResourceManager.updateZkMetadata(existingSegmentZKMetadata)) {
                LOGGER.error("Failed to update ZK metadata for segment: {} of table: {}", segmentName,
                        offlineTableName);
            }
            throw e;
        }
    }

    private String moveSegmentToPermanentDirectory(FileUploadPathProvider provider, String tableName,
            String segmentName, File tempTarredSegmentFile) throws IOException {
        // Move tarred segment file to data directory when there is no external download URL
        File tarredSegmentFile = new File(new File(provider.getBaseDataDir(), tableName), segmentName);
        FileUtils.deleteQuietly(tarredSegmentFile);
        FileUtils.moveFile(tempTarredSegmentFile, tarredSegmentFile);
        return ControllerConf.constructDownloadUrl(tableName, segmentName, provider.getVip());
    }

    private String getDownloadUri(FileUploadDownloadClient.FileUploadType uploadType, HttpHeaders headers,
            String segmentJsonStr) throws Exception {
        switch (uploadType) {
        case URI:
            return headers.getHeaderString(FileUploadDownloadClient.CustomHeaders.DOWNLOAD_URI);
        case JSON:
            // Get segmentJsonStr
            JSONTokener tokener = new JSONTokener(segmentJsonStr);
            JSONObject segmentJson = new JSONObject(tokener);
            // Download segment from the given Uri
            return segmentJson.getString(CommonConstants.Segment.Offline.DOWNLOAD_URL);
        default:
            break;
        }
        throw new UnsupportedOperationException(
                "Not support getDownloadUri method for upload type - " + uploadType);
    }

    private org.json.JSONObject formatSegments(String tableName, CommonConstants.Helix.TableType tableType)
            throws Exception {
        org.json.JSONObject obj = new org.json.JSONObject();
        obj.put(tableType.toString(), getSegments(tableName, tableType.toString()));
        return obj;
    }

    private JSONArray getSegments(String tableName, String tableType) {

        final JSONArray ret = new JSONArray();

        String realtimeTableName = TableNameBuilder.REALTIME.tableNameWithType(tableName);
        String offlineTableName = TableNameBuilder.OFFLINE.tableNameWithType(tableName);

        String tableNameWithType;
        if (CommonConstants.Helix.TableType.valueOf(tableType).toString().equals("REALTIME")) {
            tableNameWithType = realtimeTableName;
        } else {
            tableNameWithType = offlineTableName;
        }

        List<String> segmentList = _pinotHelixResourceManager.getSegmentsFor(tableNameWithType);
        IdealState idealState = HelixHelper.getTableIdealState(_pinotHelixResourceManager.getHelixZkManager(),
                tableNameWithType);

        for (String segmentName : segmentList) {
            Map<String, String> map = idealState.getInstanceStateMap(segmentName);
            if (map == null) {
                continue;
            }
            if (!map.containsValue(PinotHelixSegmentOnlineOfflineStateModelGenerator.OFFLINE_STATE)) {
                ret.put(segmentName);
            }
        }

        return ret;
    }

    // Validate that there is one file that is in the input.
    public static boolean validateMultiPart(Map<String, List<FormDataBodyPart>> map, String segmentName) {
        boolean isGood = true;
        if (map.size() != 1) {
            LOGGER.warn("Incorrect number of multi-part elements: {} (segmentName {}). Picking one", map.size(),
                    segmentName);
            isGood = false;
        }
        List<FormDataBodyPart> bodyParts = map.get(map.keySet().iterator().next());
        if (bodyParts.size() != 1) {
            LOGGER.warn(
                    "Incorrect number of elements in list in first part: {} (segmentName {}). Picking first one",
                    bodyParts.size(), segmentName);
            isGood = false;
        }
        return isGood;
    }

    /**
     * check if the segment represented by segmentFile is within the storage quota
     * @param segmentFile untarred segment. This should not be null.
     *                    segmentFile must exist on disk and must be a directory
     * @param metadata segment metadata. This should not be null.
     * @param offlineTableConfig offline table configuration. This should not be null.
     */
    private StorageQuotaChecker.QuotaCheckerResponse checkStorageQuota(@Nonnull File segmentFile,
            @Nonnull SegmentMetadata metadata, @Nonnull TableConfig offlineTableConfig) {
        TableSizeReader tableSizeReader = new TableSizeReader(_executor, _connectionManager,
                _pinotHelixResourceManager);
        StorageQuotaChecker quotaChecker = new StorageQuotaChecker(offlineTableConfig, tableSizeReader,
                _controllerMetrics);
        String offlineTableName = TableNameBuilder.OFFLINE.tableNameWithType(metadata.getTableName());
        return quotaChecker.isSegmentStorageWithinQuota(segmentFile, offlineTableName, metadata.getName(),
                _controllerConf.getServerAdminRequestTimeoutSeconds() * 1000);
    }

    /**
     * Returns true if:
     * - Segment does not have a start/end time, OR
     * - The start/end time are in a valid range (Jan 01 1971 - Jan 01, 2071)
     * @param metadata Segment metadata
     * @return
     */
    private boolean isSegmentTimeValid(SegmentMetadata metadata) {
        Interval interval = metadata.getTimeInterval();
        if (interval == null) {
            return true;
        }

        long startMillis = interval.getStartMillis();
        long endMillis = interval.getEndMillis();

        if (!TimeUtils.timeValueInValidRange(startMillis) || !TimeUtils.timeValueInValidRange(endMillis)) {
            Date minDate = new Date(TimeUtils.getValidMinTimeMillis());
            Date maxDate = new Date(TimeUtils.getValidMaxTimeMillis());

            LOGGER.error(
                    "Invalid start time '{}ms' or end time '{}ms' for segment {}, must be between '{}' and '{}' (timecolumn {}, timeunit {})",
                    interval.getStartMillis(), interval.getEndMillis(), metadata.getName(), minDate, maxDate,
                    metadata.getTimeColumn(), metadata.getTimeUnit().toString());
            return false;
        }

        return true;
    }
}