org.apache.hadoop.hdfs.server.namenode.IngestLocal.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.hdfs.server.namenode.IngestLocal.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hdfs.server.namenode;

import java.io.IOException;
import java.io.EOFException;
import java.io.File;
import java.io.FileOutputStream;
import java.io.DataOutputStream;
import java.io.BufferedInputStream;
import java.io.InputStream;
import java.io.FileInputStream;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.DataInputStream;
import java.io.RandomAccessFile;
import java.nio.channels.FileChannel;
import java.lang.Thread;
import java.util.Date;
import java.util.Collection;
import java.text.SimpleDateFormat;

import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.io.*;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.permission.*;
import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.hdfs.protocol.AvatarProtocol;
import org.apache.hadoop.hdfs.protocol.AvatarConstants.Avatar;
import org.apache.hadoop.hdfs.protocol.AvatarConstants.InstanceId;
import org.apache.hadoop.hdfs.server.namenode.AvatarNode;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.FSImage;
import org.apache.hadoop.hdfs.server.namenode.FSEditLog;
import org.apache.hadoop.hdfs.server.common.Storage;

/**
 * This class reads transaction logs from the primary's shared device
 * and feeds it to the standby NameNode.
 */

public class IngestLocal implements Runnable {

    public static final Log LOG = AvatarNode.LOG;
    static final SimpleDateFormat DATE_FORM = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");

    private StandbyNew standby;
    private Configuration confg; // configuration of local namenode
    private File ingestFile;
    volatile private boolean running = true;
    volatile private boolean lastScan = false; // is this the last scan?
    private CheckpointSignature lastSignature;
    volatile private boolean success = false; // not successfully ingested yet
    RandomAccessFile rp; // the file to read transactions from.
    FileInputStream fp;
    FileChannel fc;
    long currentPosition; // current offset in the transaction log

    IngestLocal(StandbyNew standby, Configuration conf, File edits) throws IOException {
        this.standby = standby;
        this.confg = conf;
        this.ingestFile = edits;
    }

    public void run() {

        while (running) {
            try {
                // keep ingesting transactions from remote edits log.
                loadFSEdits(ingestFile);
            } catch (Exception e) {
                LOG.warn("Ingest: Exception while processing transactions (" + running + ") "
                        + StringUtils.stringifyException(e));

                LOG.warn("Ingest: NFS has been stopped!!!! ");
                System.exit(1);
                throw new RuntimeException("Ingest: failure", e);
            } finally {
                LOG.warn("Ingest: Processing transactions has a hiccup. " + running);
            }
        }
        success = true; // successful ingest.
    }

    /**
     * Immediately Stop the ingestion of transaction logs
     */
    void stop() {
        running = false;
    }

    /**
     * Indicate that this is the last pass over the transaction log
     */
    void quiesce() {
        lastScan = true;
    }

    /**
     * Indicate that this is the last pass over the transaction log
     * Verify that file modification time of the edits log matches
     * that of the signature.
     */
    synchronized void quiesce(CheckpointSignature sig) {
        lastSignature = sig;
        lastScan = true;
    }

    private synchronized CheckpointSignature getLastCheckpointSignature() {
        return this.lastSignature;
    }

    /**
     * Indicate whether ingest was successful or not.
     * Returns true on success, else false.
     */
    boolean getIngestStatus() {
        return success;
    }

    /**
     * Load an edit log, and continue applying the changes to the in-memory 
     * structure. This is where we ingest transactions into the standby.
     */
    private int loadFSEdits(File edits) throws IOException {
        FSNamesystem fsNamesys = FSNamesystem.getFSNamesystem();
        FSDirectory fsDir = fsNamesys.dir;
        int numEdits = 0;
        int logVersion = 0;
        String clientName = null;
        String clientMachine = null;
        String path = null;
        int numOpAdd = 0, numOpClose = 0, numOpDelete = 0, numOpRename = 0, numOpSetRepl = 0, numOpMkDir = 0,
                numOpSetPerm = 0, numOpSetOwner = 0, numOpSetGenStamp = 0, numOpTimes = 0, numOpOther = 0;
        long startTime = FSNamesystem.now();

        LOG.info("Ingest: Consuming transactions from file " + edits + " of size " + edits.length());
        rp = new RandomAccessFile(edits, "r");
        fp = new FileInputStream(rp.getFD()); // open for reads
        fc = rp.getChannel();

        DataInputStream in = new DataInputStream(fp);
        try {
            // Read log file version. Could be missing. 
            in.mark(4);
            // If edits log is greater than 2G, available method will return negative
            // numbers, so we avoid having to call available
            boolean available = true;
            try {
                logVersion = in.readByte();
            } catch (EOFException e) {
                available = false;
            }
            if (available) {
                fc.position(0); // reset
                in = new DataInputStream(fp);
                logVersion = in.readInt();
                if (logVersion != FSConstants.LAYOUT_VERSION) // future version
                    throw new IOException("Ingest: Unexpected version of the file system log file: " + logVersion
                            + ". Current version = " + FSConstants.LAYOUT_VERSION + ".");
            }
            assert logVersion <= Storage.LAST_UPGRADABLE_LAYOUT_VERSION : "Unsupported version " + logVersion;
            currentPosition = fc.position();
            numEdits = ingestFSEdits(edits, in, logVersion); // continue to ingest 
        } finally {
            LOG.info("Ingest: Closing transactions file " + edits);
            fp.close();
        }
        LOG.info("Ingest: Edits file " + edits.getName() + " of size " + edits.length() + " edits # " + numEdits
                + " loaded in " + (FSNamesystem.now() - startTime) / 1000 + " seconds.");

        if (LOG.isDebugEnabled()) {
            LOG.debug("Ingest: numOpAdd = " + numOpAdd + " numOpClose = " + numOpClose + " numOpDelete = "
                    + numOpDelete + " numOpRename = " + numOpRename + " numOpSetRepl = " + numOpSetRepl
                    + " numOpMkDir = " + numOpMkDir + " numOpSetPerm = " + numOpSetPerm + " numOpSetOwner = "
                    + numOpSetOwner + " numOpSetGenStamp = " + numOpSetGenStamp + " numOpTimes = " + numOpTimes
                    + " numOpOther = " + numOpOther);
        }

        if (logVersion != FSConstants.LAYOUT_VERSION) // other version
            numEdits++; // save this image asap
        return numEdits;
    }

    /**
     * Continue to ingest transaction logs until the currentState is 
     * no longer INGEST. If lastScan is set to true, then we process 
     * till the end of the file and return.
     */
    int ingestFSEdits(File fname, DataInputStream in, int logVersion) throws IOException {
        FSNamesystem fsNamesys = FSNamesystem.getFSNamesystem();
        FSDirectory fsDir = fsNamesys.dir;
        int numEdits = 0;
        String clientName = null;
        String clientMachine = null;
        String path = null;
        int numOpAdd = 0, numOpClose = 0, numOpDelete = 0, numOpRename = 0, numOpSetRepl = 0, numOpMkDir = 0,
                numOpSetPerm = 0, numOpSetOwner = 0, numOpSetGenStamp = 0, numOpTimes = 0, numOpOther = 0;
        long startTime = FSNamesystem.now();
        boolean error = false;
        boolean quitAfterScan = false;

        while (running && !quitAfterScan) {

            // if the application requested that we make a final pass over 
            // the transaction log, then we remember it here. We close and
            // reopen the file to ensure that we can see all the data in the
            // file, one reason being that NFS has open-to-close cache
            // coherancy and the edit log could be stored in NFS.
            //
            if (lastScan) {
                LOG.info("Ingest: Starting last scan of transaction log " + fname);
                quitAfterScan = true;
                fp.close();
                rp = new RandomAccessFile(fname, "r");
                fp = new FileInputStream(rp.getFD()); // open for reads
                fc = rp.getChannel();

                // discard older buffers and start a fresh one.
                fc.position(currentPosition);
                in = new DataInputStream(fp);
            }

            //
            // Verify that signature of file matches. This is imporatant in the
            // case when the Primary NN was configured to write transactions to 
            // to devices (local and NFS) and the Primary had encountered errors
            // to the NFS device and has continued writing transactions to its
            // device only. In this case, the rollEditLog() RPC would return the
            // modtime of the edits file of the Primary's local device and will
            // not match with the timestamp of our local log from where we are
            // ingesting.
            //
            CheckpointSignature signature = getLastCheckpointSignature();
            if (signature != null) {
                long localtime = fname.lastModified();

                LOG.info("editLog : " + fname.getPath());
                LOG.info("editLog.lastModifiedTime : " + localtime);

                if (localtime == signature.editsTime) {
                    LOG.debug("Ingest: Matched modification time of edits log. ");
                } else if (localtime < signature.editsTime) {
                    LOG.info("Ingest: Timestamp of transaction log on local machine is " + localtime
                            + " and on remote namenode is " + signature.editsTime);
                    String msg = "Ingest: Timestamp of transaction log on local machine is "
                            + DATE_FORM.format(new Date(localtime)) + " and on remote namenode is "
                            + DATE_FORM.format(new Date(signature.editsTime));
                    LOG.info(msg);
                    throw new IOException(msg);
                } else {
                    LOG.info("Ingest: Timestamp of transaction log on local machine is " + localtime
                            + " and on remote namenode is " + signature.editsTime);
                    String msg = "Ingest: Timestamp of transaction log on localmachine is "
                            + DATE_FORM.format(new Date(localtime)) + " and on remote namenode is "
                            + DATE_FORM.format(new Date(signature.editsTime)) + ". But this can never happen.";
                    LOG.info(msg);
                    throw new IOException(msg);
                }
            }

            //
            // Process all existing transactions till end of file
            //
            while (running) {
                currentPosition = fc.position(); // record the current file offset.

                try {
                    long timestamp = 0;
                    long mtime = 0;
                    long atime = 0;
                    long blockSize = 0;
                    byte opcode = -1;
                    error = false;
                    try {
                        opcode = in.readByte();
                        if (opcode == OP_INVALID) {
                            FSNamesystem.LOG.debug("Ingest: Invalid opcode, reached end of log "
                                    + "Number of transactions found " + numEdits);
                            break; // No more transactions.
                        }
                    } catch (EOFException e) {
                        break; // No more transactions.
                    }
                    switch (opcode) {
                    case OP_ADD:
                    case OP_CLOSE: {
                        // versions > 0 support per file replication
                        // get name and replication
                        int length = in.readInt();
                        if (-7 == logVersion && length != 3 || -17 < logVersion && logVersion < -7 && length != 4
                                || logVersion <= -17 && length != 5) {
                            throw new IOException("Ingest: Incorrect data format." + " logVersion is " + logVersion
                                    + " but writables.length is " + length + ". ");
                        }
                        path = FSImage.readString(in);
                        short replication = readShort(in);
                        mtime = readLong(in);
                        if (logVersion <= -17) {
                            atime = readLong(in);
                        }
                        if (logVersion < -7) {
                            blockSize = readLong(in);
                        }
                        // get blocks
                        Block blocks[] = null;
                        if (logVersion <= -14) {
                            blocks = readBlocks(in);

                        } else {
                            BlockTwo oldblk = new BlockTwo();
                            int num = in.readInt();
                            blocks = new Block[num];
                            for (int i = 0; i < num; i++) {
                                oldblk.readFields(in);
                                blocks[i] = new Block(oldblk.blkid, oldblk.len, Block.GRANDFATHER_GENERATION_STAMP);
                            }
                        }

                        // Older versions of HDFS does not store the block size in inode.
                        // If the file has more than one block, use the size of the
                        // first block as the blocksize. Otherwise use the default
                        // block size.
                        if (-8 <= logVersion && blockSize == 0) {
                            if (blocks.length > 1) {
                                blockSize = blocks[0].getNumBytes();
                            } else {
                                long first = ((blocks.length == 1) ? blocks[0].getNumBytes() : 0);
                                blockSize = Math.max(fsNamesys.getDefaultBlockSize(), first);
                            }
                        }

                        PermissionStatus permissions = fsNamesys.getUpgradePermission();
                        if (logVersion <= -11) {
                            permissions = PermissionStatus.read(in);
                        }

                        // clientname, clientMachine and block locations of last block.
                        if (opcode == OP_ADD && logVersion <= -12) {
                            clientName = FSImage.readString(in);
                            clientMachine = FSImage.readString(in);
                            if (-13 <= logVersion) {
                                readDatanodeDescriptorArray(in);
                            }
                        } else {
                            clientName = "";
                            clientMachine = "";
                        }

                        // The open lease transaction re-creates a file if necessary.
                        // Delete the file if it already exists.
                        if (FSNamesystem.LOG.isDebugEnabled()) {
                            FSNamesystem.LOG.debug(opcode + ": " + path + " numblocks : " + blocks.length
                                    + " clientHolder " + clientName + " clientMachine " + clientMachine);
                        }

                        fsDir.unprotectedDelete(path, mtime);

                        // add to the file tree
                        INodeFile node = (INodeFile) fsDir.unprotectedAddFile(path, permissions, blocks,
                                replication, mtime, atime, blockSize);
                        if (opcode == OP_ADD) {
                            numOpAdd++;
                            //
                            // Replace current node with a INodeUnderConstruction.
                            // Recreate in-memory lease record.
                            //
                            INodeFileUnderConstruction cons = new INodeFileUnderConstruction(
                                    node.getLocalNameBytes(), node.getReplication(), node.getModificationTime(),
                                    node.getPreferredBlockSize(), node.getBlocks(), node.getPermissionStatus(),
                                    clientName, clientMachine, null);
                            fsDir.replaceNode(path, node, cons);
                            fsNamesys.leaseManager.addLease(cons.clientName, path);
                        }
                        break;
                    }
                    case OP_SET_REPLICATION: {
                        numOpSetRepl++;
                        path = FSImage.readString(in);
                        short replication = readShort(in);
                        fsDir.unprotectedSetReplication(path, replication, null);
                        break;
                    }
                    case OP_RENAME: {
                        numOpRename++;
                        int length = in.readInt();
                        if (length != 3) {
                            throw new IOException("Ingest: Incorrect data format. " + "Mkdir operation.");
                        }
                        String s = FSImage.readString(in);
                        String d = FSImage.readString(in);
                        timestamp = readLong(in);
                        FileStatus dinfo = fsDir.getFileInfo(d);
                        fsDir.unprotectedRenameTo(s, d, timestamp);
                        fsNamesys.changeLease(s, d, dinfo);
                        break;
                    }
                    case OP_DELETE: {
                        numOpDelete++;
                        int length = in.readInt();
                        if (length != 2) {
                            throw new IOException("Ingest: Incorrect data format. " + "delete operation.");
                        }
                        path = FSImage.readString(in);
                        timestamp = readLong(in);
                        fsDir.unprotectedDelete(path, timestamp);
                        break;
                    }
                    case OP_MKDIR: {
                        numOpMkDir++;
                        PermissionStatus permissions = fsNamesys.getUpgradePermission();
                        int length = in.readInt();
                        if (-17 < logVersion && length != 2 || logVersion <= -17 && length != 3) {
                            throw new IOException("Ingest: Incorrect data format. " + "Mkdir operation.");
                        }
                        path = FSImage.readString(in);
                        timestamp = readLong(in);

                        // The disk format stores atimes for directories as well.
                        // However, currently this is not being updated/used because of
                        // performance reasons.
                        if (logVersion <= -17) {
                            atime = readLong(in);
                        }

                        if (logVersion <= -11) {
                            permissions = PermissionStatus.read(in);
                        }
                        fsDir.unprotectedMkdir(path, permissions, timestamp);
                        break;
                    }
                    case OP_SET_GENSTAMP: {
                        numOpSetGenStamp++;
                        long lw = in.readLong();
                        fsDir.namesystem.setGenerationStamp(lw);
                        break;
                    }
                    case OP_DATANODE_ADD: {
                        numOpOther++;
                        FSImage.DatanodeImage nodeimage = new FSImage.DatanodeImage();
                        nodeimage.readFields(in);
                        //Datnodes are not persistent any more.
                        break;
                    }
                    case OP_DATANODE_REMOVE: {
                        numOpOther++;
                        DatanodeID nodeID = new DatanodeID();
                        nodeID.readFields(in);

                        //Datanodes are not persistent any more.
                        break;
                    }
                    case OP_SET_PERMISSIONS: {
                        numOpSetPerm++;
                        if (logVersion > -11)
                            throw new IOException(
                                    "Ingest: Unexpected opcode " + opcode + " for version " + logVersion);
                        fsDir.unprotectedSetPermission(FSImage.readString(in), FsPermission.read(in));
                        break;
                    }
                    case OP_SET_OWNER: {
                        numOpSetOwner++;
                        if (logVersion > -11)
                            throw new IOException(
                                    "Ingest: Unexpected opcode " + opcode + " for version " + logVersion);
                        fsDir.unprotectedSetOwner(FSImage.readString(in), FSImage.readString_EmptyAsNull(in),
                                FSImage.readString_EmptyAsNull(in));
                        break;
                    }
                    case OP_SET_NS_QUOTA: {
                        if (logVersion > -16) {
                            throw new IOException(
                                    "Ingest: Unexpected opcode " + opcode + " for version " + logVersion);
                        }
                        fsDir.unprotectedSetQuota(FSImage.readString(in), readLongWritable(in),
                                FSConstants.QUOTA_DONT_SET);
                        break;
                    }
                    case OP_CLEAR_NS_QUOTA: {
                        if (logVersion > -16) {
                            throw new IOException(
                                    "Ingest: Unexpected opcode " + opcode + " for version " + logVersion);
                        }
                        fsDir.unprotectedSetQuota(FSImage.readString(in), FSConstants.QUOTA_RESET,
                                FSConstants.QUOTA_DONT_SET);
                        break;
                    }

                    case OP_SET_QUOTA:
                        fsDir.unprotectedSetQuota(FSImage.readString(in), readLongWritable(in),
                                readLongWritable(in));

                        break;

                    case OP_TIMES: {
                        numOpTimes++;
                        int length = in.readInt();
                        if (length != 3) {
                            throw new IOException("Ingest: Incorrect data format. " + "times operation.");
                        }
                        path = FSImage.readString(in);
                        mtime = readLong(in);
                        atime = readLong(in);
                        fsDir.unprotectedSetTimes(path, mtime, atime, true);
                        break;
                    }
                    default: {
                        throw new IOException("Ingest: Never seen opcode " + opcode);
                    }
                    }
                    numEdits++;
                    LOG.info("Ingest: Processed transaction from " + fname + " opcode " + opcode + " file offset "
                            + currentPosition);
                } //rty---
                catch (IOException e) {
                    error = true; // if we haven't reached eof, then error.
                    break;
                }
            } // while (running)-----

            // if we failed to read the entire transaction from disk, 
            // then roll back to the offset where there was a last good 
            // read, sleep for sometime for new transaction to
            // appear in the file and then continue;
            //
            if (error || running) {

                // discard older buffers and start a fresh one.
                fc.position(currentPosition);
                in = new DataInputStream(fp);

                if (error) {
                    LOG.info("Ingest: Incomplete transaction record at offset " + fc.position()
                            + " but the file is of size " + fc.size() + ". Continuing....");
                }

                if (running && !lastScan) {
                    try {
                        Thread.sleep(1000); // sleep for a second
                    } catch (InterruptedException e) {
                        // break out of waiting if we receive an interrupt.
                    }
                }
            }
        } //while (running && !quitAfterScan)-------------
        LOG.info("Ingest: Edits file " + fname.getName() + " numedits " + numEdits + " loaded in "
                + (FSNamesystem.now() - startTime) / 1000 + " seconds.");

        // If the last Scan was completed, then stop the Ingest thread.
        if (lastScan && quitAfterScan) {
            LOG.info("Ingest: lastScan completed.");
            running = false;
        }
        return numEdits; // total transactions consumed
    }

    // a place holder for reading a long
    private static final LongWritable longWritable = new LongWritable();

    /** Read an integer from an input stream */
    private static long readLongWritable(DataInputStream in) throws IOException {
        synchronized (longWritable) {
            longWritable.readFields(in);
            return longWritable.get();
        }
    }

    /**
     * A class to read in blocks stored in the old format. The only two
     * fields in the block were blockid and length.
     */
    static class BlockTwo implements Writable {
        long blkid;
        long len;

        static { // register a ctor
            WritableFactories.setFactory(BlockTwo.class, new WritableFactory() {
                public Writable newInstance() {
                    return new BlockTwo();
                }
            });
        }

        BlockTwo() {
            blkid = 0;
            len = 0;
        }

        /////////////////////////////////////
        // Writable
        /////////////////////////////////////
        public void write(DataOutput out) throws IOException {
            out.writeLong(blkid);
            out.writeLong(len);
        }

        public void readFields(DataInput in) throws IOException {
            this.blkid = in.readLong();
            this.len = in.readLong();
        }
    }

    //
    // These methods are copied from FsEdits/FsImage because they are defined as
    // private scope in those files.
    //
    static private DatanodeDescriptor[] readDatanodeDescriptorArray(DataInput in) throws IOException {
        DatanodeDescriptor[] locations = new DatanodeDescriptor[in.readInt()];
        for (int i = 0; i < locations.length; i++) {
            locations[i] = new DatanodeDescriptor();
            locations[i].readFieldsFromFSEditLog(in);
        }
        return locations;
    }

    static private short readShort(DataInputStream in) throws IOException {
        return Short.parseShort(FSImage.readString(in));
    }

    static private long readLong(DataInputStream in) throws IOException {
        return Long.parseLong(FSImage.readString(in));
    }

    static private Block[] readBlocks(DataInputStream in) throws IOException {
        int numBlocks = in.readInt();
        Block[] blocks = new Block[numBlocks];
        for (int i = 0; i < numBlocks; i++) {
            blocks[i] = new Block();
            blocks[i].readFields(in);
        }
        return blocks;
    }

    private static final byte OP_INVALID = -1;
    private static final byte OP_ADD = 0;
    private static final byte OP_RENAME = 1; // rename
    private static final byte OP_DELETE = 2; // delete
    private static final byte OP_MKDIR = 3; // create directory
    private static final byte OP_SET_REPLICATION = 4; // set replication
    //the following two are used only for backward compatibility :
    @Deprecated
    private static final byte OP_DATANODE_ADD = 5;
    @Deprecated
    private static final byte OP_DATANODE_REMOVE = 6;
    private static final byte OP_SET_PERMISSIONS = 7;
    private static final byte OP_SET_OWNER = 8;
    private static final byte OP_CLOSE = 9; // close after write
    private static final byte OP_SET_GENSTAMP = 10; // store genstamp
    /* The following two are not used any more. Should be removed once
     * LAST_UPGRADABLE_LAYOUT_VERSION is -17 or newer. */
    private static final byte OP_SET_NS_QUOTA = 11; // set namespace quota
    private static final byte OP_CLEAR_NS_QUOTA = 12; // clear namespace quota
    private static final byte OP_TIMES = 13; // sets mod & access time on a file
    private static final byte OP_SET_QUOTA = 14; // sets name and disk quotas.
}