Java tutorial
/* * Copyright 2015 gitblit.com. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.gitblit.manager; import java.io.EOFException; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.RandomAccessFile; import java.lang.reflect.Type; import java.nio.file.Files; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collection; import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.gitblit.IStoredSettings; import com.gitblit.Keys; import com.gitblit.models.FilestoreModel; import com.gitblit.models.FilestoreModel.Status; import com.gitblit.models.RepositoryModel; import com.gitblit.models.UserModel; import com.gitblit.utils.ArrayUtils; import com.gitblit.utils.JsonUtils.GmtDateTypeAdapter; import com.google.gson.ExclusionStrategy; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.reflect.TypeToken; import com.google.inject.Inject; import com.google.inject.Singleton; /** * FilestoreManager handles files uploaded via: * + git-lfs * + ticket attachment (TBD) * * Files are stored using their SHA256 hash (as per git-lfs) * If the same file is uploaded through different repositories no additional space is used * Access is controlled through the current repository permissions. * * TODO: Identify what and how the actual BLOBs should work with federation * * @author Paul Martin * */ @Singleton public class FilestoreManager implements IFilestoreManager { private final Logger logger = LoggerFactory.getLogger(getClass()); private final IRuntimeManager runtimeManager; private final IRepositoryManager repositoryManager; private final IStoredSettings settings; public static final int UNDEFINED_SIZE = -1; private static final String METAFILE = "filestore.json"; private static final String METAFILE_TMP = "filestore.json.tmp"; protected static final Type METAFILE_TYPE = new TypeToken<Collection<FilestoreModel>>() { }.getType(); private Map<String, FilestoreModel> fileCache = new ConcurrentHashMap<String, FilestoreModel>(); @Inject FilestoreManager(IRuntimeManager runtimeManager, IRepositoryManager repositoryManager) { this.runtimeManager = runtimeManager; this.repositoryManager = repositoryManager; this.settings = runtimeManager.getSettings(); } @Override public IManager start() { // Try to load any existing metadata File dir = getStorageFolder(); dir.mkdirs(); File metadata = new File(dir, METAFILE); if (metadata.exists()) { Collection<FilestoreModel> items = null; Gson gson = gson(); try (FileReader file = new FileReader(metadata)) { items = gson.fromJson(file, METAFILE_TYPE); file.close(); } catch (IOException e) { e.printStackTrace(); } for (Iterator<FilestoreModel> itr = items.iterator(); itr.hasNext();) { FilestoreModel model = itr.next(); fileCache.put(model.oid, model); } logger.info("Loaded {} items from filestore metadata file", fileCache.size()); } else { logger.info("No filestore metadata file found"); } return this; } @Override public IManager stop() { return this; } @Override public boolean isValidOid(String oid) { //NOTE: Assuming SHA256 support only as per git-lfs return Pattern.matches("[a-fA-F0-9]{64}", oid); } @Override public FilestoreModel.Status addObject(String oid, long size, UserModel user, RepositoryModel repo) { //Handle access control if (!user.canPush(repo)) { if (user == UserModel.ANONYMOUS) { return Status.AuthenticationRequired; } else { return Status.Error_Unauthorized; } } //Handle object details if (!isValidOid(oid)) { return Status.Error_Invalid_Oid; } if (fileCache.containsKey(oid)) { FilestoreModel item = fileCache.get(oid); if (!item.isInErrorState() && (size != UNDEFINED_SIZE) && (item.getSize() != size)) { return Status.Error_Size_Mismatch; } item.addRepository(repo.name); if (item.isInErrorState()) { item.reset(user, size); } } else { if (size < 0) { return Status.Error_Invalid_Size; } if ((getMaxUploadSize() != UNDEFINED_SIZE) && (size > getMaxUploadSize())) { return Status.Error_Exceeds_Size_Limit; } FilestoreModel model = new FilestoreModel(oid, size, user, repo.name); fileCache.put(oid, model); saveFilestoreModel(model); } return fileCache.get(oid).getStatus(); } @Override public FilestoreModel.Status uploadBlob(String oid, long size, UserModel user, RepositoryModel repo, InputStream streamIn) { //Access control and object logic Status state = addObject(oid, size, user, repo); if (state != Status.Upload_Pending) { return state; } FilestoreModel model = fileCache.get(oid); if (!model.actionUpload(user)) { return Status.Upload_In_Progress; } else { long actualSize = 0; File file = getStoragePath(oid); try { file.getParentFile().mkdirs(); file.createNewFile(); try (FileOutputStream streamOut = new FileOutputStream(file)) { actualSize = IOUtils.copyLarge(streamIn, streamOut); streamOut.flush(); streamOut.close(); if (model.getSize() != actualSize) { model.setStatus(Status.Error_Size_Mismatch, user); logger.warn(MessageFormat.format( "Failed to upload blob {0} due to size mismatch, expected {1} got {2}", oid, model.getSize(), actualSize)); } else { String actualOid = ""; try (FileInputStream fileForHash = new FileInputStream(file)) { actualOid = DigestUtils.sha256Hex(fileForHash); fileForHash.close(); } if (oid.equalsIgnoreCase(actualOid)) { model.setStatus(Status.Available, user); } else { model.setStatus(Status.Error_Hash_Mismatch, user); logger.warn(MessageFormat.format( "Failed to upload blob {0} due to hash mismatch, got {1}", oid, actualOid)); } } } } catch (Exception e) { model.setStatus(Status.Error_Unknown, user); logger.warn(MessageFormat.format("Failed to upload blob {0}", oid), e); } finally { saveFilestoreModel(model); } if (model.isInErrorState()) { file.delete(); model.removeRepository(repo.name); } } return model.getStatus(); } private FilestoreModel.Status canGetObject(String oid, UserModel user, RepositoryModel repo) { //Access Control if (!user.canView(repo)) { if (user == UserModel.ANONYMOUS) { return Status.AuthenticationRequired; } else { return Status.Error_Unauthorized; } } //Object Logic if (!isValidOid(oid)) { return Status.Error_Invalid_Oid; } if (!fileCache.containsKey(oid)) { return Status.Unavailable; } FilestoreModel item = fileCache.get(oid); if (item.getStatus() == Status.Available) { return Status.Available; } return Status.Unavailable; } @Override public FilestoreModel getObject(String oid, UserModel user, RepositoryModel repo) { if (canGetObject(oid, user, repo) == Status.Available) { return fileCache.get(oid); } return null; } @Override public FilestoreModel.Status downloadBlob(String oid, UserModel user, RepositoryModel repo, OutputStream streamOut) { //Access control and object logic Status status = canGetObject(oid, user, repo); if (status != Status.Available) { return status; } FilestoreModel item = fileCache.get(oid); if (streamOut != null) { try (FileInputStream streamIn = new FileInputStream(getStoragePath(oid))) { IOUtils.copyLarge(streamIn, streamOut); streamOut.flush(); streamIn.close(); } catch (EOFException e) { logger.error(MessageFormat.format("Client aborted connection for {0}", oid), e); return Status.Error_Unexpected_Stream_End; } catch (Exception e) { logger.error(MessageFormat.format("Failed to download blob {0}", oid), e); return Status.Error_Unknown; } } return item.getStatus(); } @Override public List<FilestoreModel> getAllObjects(UserModel user) { final List<RepositoryModel> viewableRepositories = repositoryManager.getRepositoryModels(user); List<String> viewableRepositoryNames = new ArrayList<String>(viewableRepositories.size()); for (RepositoryModel repository : viewableRepositories) { viewableRepositoryNames.add(repository.name); } if (viewableRepositoryNames.size() == 0) { return null; } final Collection<FilestoreModel> allFiles = fileCache.values(); List<FilestoreModel> userViewableFiles = new ArrayList<FilestoreModel>(allFiles.size()); for (FilestoreModel file : allFiles) { if (file.isInRepositoryList(viewableRepositoryNames)) { userViewableFiles.add(file); } } return userViewableFiles; } @Override public File getStorageFolder() { return runtimeManager.getFileOrFolder(Keys.filestore.storageFolder, "${baseFolder}/lfs"); } @Override public File getStoragePath(String oid) { return new File(getStorageFolder(), oid.substring(0, 2).concat("/").concat(oid.substring(2))); } @Override public long getMaxUploadSize() { return settings.getLong(Keys.filestore.maxUploadSize, -1); } @Override public long getFilestoreUsedByteCount() { Iterator<FilestoreModel> iterator = fileCache.values().iterator(); long total = 0; while (iterator.hasNext()) { FilestoreModel item = iterator.next(); if (item.getStatus() == Status.Available) { total += item.getSize(); } } return total; } @Override public long getFilestoreAvailableByteCount() { try { return Files.getFileStore(getStorageFolder().toPath()).getUsableSpace(); } catch (IOException e) { logger.error(MessageFormat.format("Failed to retrive available space in Filestore {0}", e)); } return UNDEFINED_SIZE; }; private synchronized void saveFilestoreModel(FilestoreModel model) { File metaFile = new File(getStorageFolder(), METAFILE); File metaFileTmp = new File(getStorageFolder(), METAFILE_TMP); boolean isNewFile = false; try { if (!metaFile.exists()) { metaFile.getParentFile().mkdirs(); metaFile.createNewFile(); isNewFile = true; } FileUtils.copyFile(metaFile, metaFileTmp); } catch (IOException e) { logger.error("Writing filestore model to file {0}, {1}", METAFILE, e); } try (RandomAccessFile fs = new RandomAccessFile(metaFileTmp, "rw")) { if (isNewFile) { fs.writeBytes("["); } else { fs.seek(fs.length() - 1); fs.writeBytes(","); } fs.writeBytes(gson().toJson(model)); fs.writeBytes("]"); fs.close(); } catch (IOException e) { logger.error("Writing filestore model to file {0}, {1}", METAFILE_TMP, e); } try { if (metaFileTmp.exists()) { FileUtils.copyFile(metaFileTmp, metaFile); metaFileTmp.delete(); } else { logger.error("Writing filestore model to file {0}", METAFILE); } } catch (IOException e) { logger.error("Writing filestore model to file {0}, {1}", METAFILE, e); } } /* * Intended for testing purposes only */ @Override public void clearFilestoreCache() { fileCache.clear(); } private static Gson gson(ExclusionStrategy... strategies) { GsonBuilder builder = new GsonBuilder(); builder.registerTypeAdapter(Date.class, new GmtDateTypeAdapter()); if (!ArrayUtils.isEmpty(strategies)) { builder.setExclusionStrategies(strategies); } return builder.create(); } }