diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/TraceRepo.java b/core/src/main/java/org/apache/accumulo/core/fate/TraceRepo.java similarity index 93% rename from server/manager/src/main/java/org/apache/accumulo/manager/tableOps/TraceRepo.java rename to core/src/main/java/org/apache/accumulo/core/fate/TraceRepo.java index f9da68dccad..f9c709eb239 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/TraceRepo.java +++ b/core/src/main/java/org/apache/accumulo/core/fate/TraceRepo.java @@ -16,11 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.accumulo.manager.tableOps; +package org.apache.accumulo.core.fate; import org.apache.accumulo.core.clientImpl.thrift.TInfo; -import org.apache.accumulo.core.fate.FateId; -import org.apache.accumulo.core.fate.Repo; import org.apache.accumulo.core.trace.TraceUtil; import io.opentelemetry.api.trace.Span; @@ -107,12 +105,16 @@ public String getReturn() { return repo.getReturn(); } + public Repo getWrapped() { + return repo; + } + /** * @return string version of Repo that is suitable for logging */ - public static String toLogString(Repo repo) { + public static String toLogString(Repo repo) { if (repo instanceof TraceRepo) { - repo = ((TraceRepo) repo).repo; + repo = ((TraceRepo) repo).repo; } return repo.getClass() + " " + repo.getName(); diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/manager/thrift/BulkImportState.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/manager/thrift/BulkImportState.java deleted file mode 100644 index 3cb402c314d..00000000000 --- a/core/src/main/thrift-gen-java/org/apache/accumulo/core/manager/thrift/BulkImportState.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/** - * Autogenerated by Thrift Compiler (0.17.0) - * - * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - * @generated - */ -package org.apache.accumulo.core.manager.thrift; - - -public enum BulkImportState implements org.apache.thrift.TEnum { - INITIAL(0), - MOVING(1), - PROCESSING(2), - ASSIGNING(3), - LOADING(4), - COPY_FILES(5), - CLEANUP(6); - - private final int value; - - private BulkImportState(int value) { - this.value = value; - } - - /** - * Get the integer value of this enum value, as defined in the Thrift IDL. - */ - @Override - public int getValue() { - return value; - } - - /** - * Find a the enum type by its integer value, as defined in the Thrift IDL. - * @return null if the value is not found. - */ - @org.apache.thrift.annotation.Nullable - public static BulkImportState findByValue(int value) { - switch (value) { - case 0: - return INITIAL; - case 1: - return MOVING; - case 2: - return PROCESSING; - case 3: - return ASSIGNING; - case 4: - return LOADING; - case 5: - return COPY_FILES; - case 6: - return CLEANUP; - default: - return null; - } - } -} diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/manager/thrift/BulkImportStatus.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/manager/thrift/BulkImportStatus.java deleted file mode 100644 index 64de85baf3c..00000000000 --- a/core/src/main/thrift-gen-java/org/apache/accumulo/core/manager/thrift/BulkImportStatus.java +++ /dev/null @@ -1,627 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/** - * Autogenerated by Thrift Compiler (0.17.0) - * - * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - * @generated - */ -package org.apache.accumulo.core.manager.thrift; - -@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) -public class BulkImportStatus implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("BulkImportStatus"); - - private static final org.apache.thrift.protocol.TField START_TIME_FIELD_DESC = new org.apache.thrift.protocol.TField("startTime", org.apache.thrift.protocol.TType.I64, (short)1); - private static final org.apache.thrift.protocol.TField FILENAME_FIELD_DESC = new org.apache.thrift.protocol.TField("filename", org.apache.thrift.protocol.TType.STRING, (short)2); - private static final org.apache.thrift.protocol.TField STATE_FIELD_DESC = new org.apache.thrift.protocol.TField("state", org.apache.thrift.protocol.TType.I32, (short)3); - - private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new BulkImportStatusStandardSchemeFactory(); - private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new BulkImportStatusTupleSchemeFactory(); - - public long startTime; // required - public @org.apache.thrift.annotation.Nullable java.lang.String filename; // required - /** - * - * @see BulkImportState - */ - public @org.apache.thrift.annotation.Nullable BulkImportState state; // required - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { - START_TIME((short)1, "startTime"), - FILENAME((short)2, "filename"), - /** - * - * @see BulkImportState - */ - STATE((short)3, "state"); - - private static final java.util.Map byName = new java.util.HashMap(); - - static { - for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } - } - - /** - * Find the _Fields constant that matches fieldId, or null if its not found. - */ - @org.apache.thrift.annotation.Nullable - public static _Fields findByThriftId(int fieldId) { - switch(fieldId) { - case 1: // START_TIME - return START_TIME; - case 2: // FILENAME - return FILENAME; - case 3: // STATE - return STATE; - default: - return null; - } - } - - /** - * Find the _Fields constant that matches fieldId, throwing an exception - * if it is not found. - */ - public static _Fields findByThriftIdOrThrow(int fieldId) { - _Fields fields = findByThriftId(fieldId); - if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); - return fields; - } - - /** - * Find the _Fields constant that matches name, or null if its not found. - */ - @org.apache.thrift.annotation.Nullable - public static _Fields findByName(java.lang.String name) { - return byName.get(name); - } - - private final short _thriftId; - private final java.lang.String _fieldName; - - _Fields(short thriftId, java.lang.String fieldName) { - _thriftId = thriftId; - _fieldName = fieldName; - } - - @Override - public short getThriftFieldId() { - return _thriftId; - } - - @Override - public java.lang.String getFieldName() { - return _fieldName; - } - } - - // isset id assignments - private static final int __STARTTIME_ISSET_ID = 0; - private byte __isset_bitfield = 0; - public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; - static { - java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - tmpMap.put(_Fields.START_TIME, new org.apache.thrift.meta_data.FieldMetaData("startTime", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); - tmpMap.put(_Fields.FILENAME, new org.apache.thrift.meta_data.FieldMetaData("filename", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); - tmpMap.put(_Fields.STATE, new org.apache.thrift.meta_data.FieldMetaData("state", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, BulkImportState.class))); - metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(BulkImportStatus.class, metaDataMap); - } - - public BulkImportStatus() { - } - - public BulkImportStatus( - long startTime, - java.lang.String filename, - BulkImportState state) - { - this(); - this.startTime = startTime; - setStartTimeIsSet(true); - this.filename = filename; - this.state = state; - } - - /** - * Performs a deep copy on other. - */ - public BulkImportStatus(BulkImportStatus other) { - __isset_bitfield = other.__isset_bitfield; - this.startTime = other.startTime; - if (other.isSetFilename()) { - this.filename = other.filename; - } - if (other.isSetState()) { - this.state = other.state; - } - } - - @Override - public BulkImportStatus deepCopy() { - return new BulkImportStatus(this); - } - - @Override - public void clear() { - setStartTimeIsSet(false); - this.startTime = 0; - this.filename = null; - this.state = null; - } - - public long getStartTime() { - return this.startTime; - } - - public BulkImportStatus setStartTime(long startTime) { - this.startTime = startTime; - setStartTimeIsSet(true); - return this; - } - - public void unsetStartTime() { - __isset_bitfield = org.apache.thrift.EncodingUtils.clearBit(__isset_bitfield, __STARTTIME_ISSET_ID); - } - - /** Returns true if field startTime is set (has been assigned a value) and false otherwise */ - public boolean isSetStartTime() { - return org.apache.thrift.EncodingUtils.testBit(__isset_bitfield, __STARTTIME_ISSET_ID); - } - - public void setStartTimeIsSet(boolean value) { - __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __STARTTIME_ISSET_ID, value); - } - - @org.apache.thrift.annotation.Nullable - public java.lang.String getFilename() { - return this.filename; - } - - public BulkImportStatus setFilename(@org.apache.thrift.annotation.Nullable java.lang.String filename) { - this.filename = filename; - return this; - } - - public void unsetFilename() { - this.filename = null; - } - - /** Returns true if field filename is set (has been assigned a value) and false otherwise */ - public boolean isSetFilename() { - return this.filename != null; - } - - public void setFilenameIsSet(boolean value) { - if (!value) { - this.filename = null; - } - } - - /** - * - * @see BulkImportState - */ - @org.apache.thrift.annotation.Nullable - public BulkImportState getState() { - return this.state; - } - - /** - * - * @see BulkImportState - */ - public BulkImportStatus setState(@org.apache.thrift.annotation.Nullable BulkImportState state) { - this.state = state; - return this; - } - - public void unsetState() { - this.state = null; - } - - /** Returns true if field state is set (has been assigned a value) and false otherwise */ - public boolean isSetState() { - return this.state != null; - } - - public void setStateIsSet(boolean value) { - if (!value) { - this.state = null; - } - } - - @Override - public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { - switch (field) { - case START_TIME: - if (value == null) { - unsetStartTime(); - } else { - setStartTime((java.lang.Long)value); - } - break; - - case FILENAME: - if (value == null) { - unsetFilename(); - } else { - setFilename((java.lang.String)value); - } - break; - - case STATE: - if (value == null) { - unsetState(); - } else { - setState((BulkImportState)value); - } - break; - - } - } - - @org.apache.thrift.annotation.Nullable - @Override - public java.lang.Object getFieldValue(_Fields field) { - switch (field) { - case START_TIME: - return getStartTime(); - - case FILENAME: - return getFilename(); - - case STATE: - return getState(); - - } - throw new java.lang.IllegalStateException(); - } - - /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ - @Override - public boolean isSet(_Fields field) { - if (field == null) { - throw new java.lang.IllegalArgumentException(); - } - - switch (field) { - case START_TIME: - return isSetStartTime(); - case FILENAME: - return isSetFilename(); - case STATE: - return isSetState(); - } - throw new java.lang.IllegalStateException(); - } - - @Override - public boolean equals(java.lang.Object that) { - if (that instanceof BulkImportStatus) - return this.equals((BulkImportStatus)that); - return false; - } - - public boolean equals(BulkImportStatus that) { - if (that == null) - return false; - if (this == that) - return true; - - boolean this_present_startTime = true; - boolean that_present_startTime = true; - if (this_present_startTime || that_present_startTime) { - if (!(this_present_startTime && that_present_startTime)) - return false; - if (this.startTime != that.startTime) - return false; - } - - boolean this_present_filename = true && this.isSetFilename(); - boolean that_present_filename = true && that.isSetFilename(); - if (this_present_filename || that_present_filename) { - if (!(this_present_filename && that_present_filename)) - return false; - if (!this.filename.equals(that.filename)) - return false; - } - - boolean this_present_state = true && this.isSetState(); - boolean that_present_state = true && that.isSetState(); - if (this_present_state || that_present_state) { - if (!(this_present_state && that_present_state)) - return false; - if (!this.state.equals(that.state)) - return false; - } - - return true; - } - - @Override - public int hashCode() { - int hashCode = 1; - - hashCode = hashCode * 8191 + org.apache.thrift.TBaseHelper.hashCode(startTime); - - hashCode = hashCode * 8191 + ((isSetFilename()) ? 131071 : 524287); - if (isSetFilename()) - hashCode = hashCode * 8191 + filename.hashCode(); - - hashCode = hashCode * 8191 + ((isSetState()) ? 131071 : 524287); - if (isSetState()) - hashCode = hashCode * 8191 + state.getValue(); - - return hashCode; - } - - @Override - public int compareTo(BulkImportStatus other) { - if (!getClass().equals(other.getClass())) { - return getClass().getName().compareTo(other.getClass().getName()); - } - - int lastComparison = 0; - - lastComparison = java.lang.Boolean.compare(isSetStartTime(), other.isSetStartTime()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetStartTime()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.startTime, other.startTime); - if (lastComparison != 0) { - return lastComparison; - } - } - lastComparison = java.lang.Boolean.compare(isSetFilename(), other.isSetFilename()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetFilename()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.filename, other.filename); - if (lastComparison != 0) { - return lastComparison; - } - } - lastComparison = java.lang.Boolean.compare(isSetState(), other.isSetState()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetState()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.state, other.state); - if (lastComparison != 0) { - return lastComparison; - } - } - return 0; - } - - @org.apache.thrift.annotation.Nullable - @Override - public _Fields fieldForId(int fieldId) { - return _Fields.findByThriftId(fieldId); - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { - scheme(iprot).read(iprot, this); - } - - @Override - public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { - scheme(oprot).write(oprot, this); - } - - @Override - public java.lang.String toString() { - java.lang.StringBuilder sb = new java.lang.StringBuilder("BulkImportStatus("); - boolean first = true; - - sb.append("startTime:"); - sb.append(this.startTime); - first = false; - if (!first) sb.append(", "); - sb.append("filename:"); - if (this.filename == null) { - sb.append("null"); - } else { - sb.append(this.filename); - } - first = false; - if (!first) sb.append(", "); - sb.append("state:"); - if (this.state == null) { - sb.append("null"); - } else { - sb.append(this.state); - } - first = false; - sb.append(")"); - return sb.toString(); - } - - public void validate() throws org.apache.thrift.TException { - // check for required fields - // check for sub-struct validity - } - - private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { - try { - write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { - try { - // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. - __isset_bitfield = 0; - read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private static class BulkImportStatusStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { - @Override - public BulkImportStatusStandardScheme getScheme() { - return new BulkImportStatusStandardScheme(); - } - } - - private static class BulkImportStatusStandardScheme extends org.apache.thrift.scheme.StandardScheme { - - @Override - public void read(org.apache.thrift.protocol.TProtocol iprot, BulkImportStatus struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TField schemeField; - iprot.readStructBegin(); - while (true) - { - schemeField = iprot.readFieldBegin(); - if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { - break; - } - switch (schemeField.id) { - case 1: // START_TIME - if (schemeField.type == org.apache.thrift.protocol.TType.I64) { - struct.startTime = iprot.readI64(); - struct.setStartTimeIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - case 2: // FILENAME - if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.filename = iprot.readString(); - struct.setFilenameIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - case 3: // STATE - if (schemeField.type == org.apache.thrift.protocol.TType.I32) { - struct.state = org.apache.accumulo.core.manager.thrift.BulkImportState.findByValue(iprot.readI32()); - struct.setStateIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - default: - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - iprot.readFieldEnd(); - } - iprot.readStructEnd(); - - // check for required fields of primitive type, which can't be checked in the validate method - struct.validate(); - } - - @Override - public void write(org.apache.thrift.protocol.TProtocol oprot, BulkImportStatus struct) throws org.apache.thrift.TException { - struct.validate(); - - oprot.writeStructBegin(STRUCT_DESC); - oprot.writeFieldBegin(START_TIME_FIELD_DESC); - oprot.writeI64(struct.startTime); - oprot.writeFieldEnd(); - if (struct.filename != null) { - oprot.writeFieldBegin(FILENAME_FIELD_DESC); - oprot.writeString(struct.filename); - oprot.writeFieldEnd(); - } - if (struct.state != null) { - oprot.writeFieldBegin(STATE_FIELD_DESC); - oprot.writeI32(struct.state.getValue()); - oprot.writeFieldEnd(); - } - oprot.writeFieldStop(); - oprot.writeStructEnd(); - } - - } - - private static class BulkImportStatusTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { - @Override - public BulkImportStatusTupleScheme getScheme() { - return new BulkImportStatusTupleScheme(); - } - } - - private static class BulkImportStatusTupleScheme extends org.apache.thrift.scheme.TupleScheme { - - @Override - public void write(org.apache.thrift.protocol.TProtocol prot, BulkImportStatus struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet optionals = new java.util.BitSet(); - if (struct.isSetStartTime()) { - optionals.set(0); - } - if (struct.isSetFilename()) { - optionals.set(1); - } - if (struct.isSetState()) { - optionals.set(2); - } - oprot.writeBitSet(optionals, 3); - if (struct.isSetStartTime()) { - oprot.writeI64(struct.startTime); - } - if (struct.isSetFilename()) { - oprot.writeString(struct.filename); - } - if (struct.isSetState()) { - oprot.writeI32(struct.state.getValue()); - } - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol prot, BulkImportStatus struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(3); - if (incoming.get(0)) { - struct.startTime = iprot.readI64(); - struct.setStartTimeIsSet(true); - } - if (incoming.get(1)) { - struct.filename = iprot.readString(); - struct.setFilenameIsSet(true); - } - if (incoming.get(2)) { - struct.state = org.apache.accumulo.core.manager.thrift.BulkImportState.findByValue(iprot.readI32()); - struct.setStateIsSet(true); - } - } - } - - private static S scheme(org.apache.thrift.protocol.TProtocol proto) { - return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); - } - private static void unusedMethod() {} -} - diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/manager/thrift/FateService.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/manager/thrift/FateService.java index 9fd6e7808f2..c80d7e165e0 100644 --- a/core/src/main/thrift-gen-java/org/apache/accumulo/core/manager/thrift/FateService.java +++ b/core/src/main/thrift-gen-java/org/apache/accumulo/core/manager/thrift/FateService.java @@ -3182,13 +3182,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, executeFateOperatio case 5: // ARGUMENTS if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { { - org.apache.thrift.protocol.TList _list78 = iprot.readListBegin(); - struct.arguments = new java.util.ArrayList(_list78.size); - @org.apache.thrift.annotation.Nullable java.nio.ByteBuffer _elem79; - for (int _i80 = 0; _i80 < _list78.size; ++_i80) + org.apache.thrift.protocol.TList _list70 = iprot.readListBegin(); + struct.arguments = new java.util.ArrayList(_list70.size); + @org.apache.thrift.annotation.Nullable java.nio.ByteBuffer _elem71; + for (int _i72 = 0; _i72 < _list70.size; ++_i72) { - _elem79 = iprot.readBinary(); - struct.arguments.add(_elem79); + _elem71 = iprot.readBinary(); + struct.arguments.add(_elem71); } iprot.readListEnd(); } @@ -3200,15 +3200,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, executeFateOperatio case 6: // OPTIONS if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { { - org.apache.thrift.protocol.TMap _map81 = iprot.readMapBegin(); - struct.options = new java.util.HashMap(2*_map81.size); - @org.apache.thrift.annotation.Nullable java.lang.String _key82; - @org.apache.thrift.annotation.Nullable java.lang.String _val83; - for (int _i84 = 0; _i84 < _map81.size; ++_i84) + org.apache.thrift.protocol.TMap _map73 = iprot.readMapBegin(); + struct.options = new java.util.HashMap(2*_map73.size); + @org.apache.thrift.annotation.Nullable java.lang.String _key74; + @org.apache.thrift.annotation.Nullable java.lang.String _val75; + for (int _i76 = 0; _i76 < _map73.size; ++_i76) { - _key82 = iprot.readString(); - _val83 = iprot.readString(); - struct.options.put(_key82, _val83); + _key74 = iprot.readString(); + _val75 = iprot.readString(); + struct.options.put(_key74, _val75); } iprot.readMapEnd(); } @@ -3265,9 +3265,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, executeFateOperati oprot.writeFieldBegin(ARGUMENTS_FIELD_DESC); { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.arguments.size())); - for (java.nio.ByteBuffer _iter85 : struct.arguments) + for (java.nio.ByteBuffer _iter77 : struct.arguments) { - oprot.writeBinary(_iter85); + oprot.writeBinary(_iter77); } oprot.writeListEnd(); } @@ -3277,10 +3277,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, executeFateOperati oprot.writeFieldBegin(OPTIONS_FIELD_DESC); { oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.options.size())); - for (java.util.Map.Entry _iter86 : struct.options.entrySet()) + for (java.util.Map.Entry _iter78 : struct.options.entrySet()) { - oprot.writeString(_iter86.getKey()); - oprot.writeString(_iter86.getValue()); + oprot.writeString(_iter78.getKey()); + oprot.writeString(_iter78.getValue()); } oprot.writeMapEnd(); } @@ -3345,19 +3345,19 @@ public void write(org.apache.thrift.protocol.TProtocol prot, executeFateOperatio if (struct.isSetArguments()) { { oprot.writeI32(struct.arguments.size()); - for (java.nio.ByteBuffer _iter87 : struct.arguments) + for (java.nio.ByteBuffer _iter79 : struct.arguments) { - oprot.writeBinary(_iter87); + oprot.writeBinary(_iter79); } } } if (struct.isSetOptions()) { { oprot.writeI32(struct.options.size()); - for (java.util.Map.Entry _iter88 : struct.options.entrySet()) + for (java.util.Map.Entry _iter80 : struct.options.entrySet()) { - oprot.writeString(_iter88.getKey()); - oprot.writeString(_iter88.getValue()); + oprot.writeString(_iter80.getKey()); + oprot.writeString(_iter80.getValue()); } } } @@ -3391,28 +3391,28 @@ public void read(org.apache.thrift.protocol.TProtocol prot, executeFateOperation } if (incoming.get(4)) { { - org.apache.thrift.protocol.TList _list89 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRING); - struct.arguments = new java.util.ArrayList(_list89.size); - @org.apache.thrift.annotation.Nullable java.nio.ByteBuffer _elem90; - for (int _i91 = 0; _i91 < _list89.size; ++_i91) + org.apache.thrift.protocol.TList _list81 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRING); + struct.arguments = new java.util.ArrayList(_list81.size); + @org.apache.thrift.annotation.Nullable java.nio.ByteBuffer _elem82; + for (int _i83 = 0; _i83 < _list81.size; ++_i83) { - _elem90 = iprot.readBinary(); - struct.arguments.add(_elem90); + _elem82 = iprot.readBinary(); + struct.arguments.add(_elem82); } } struct.setArgumentsIsSet(true); } if (incoming.get(5)) { { - org.apache.thrift.protocol.TMap _map92 = iprot.readMapBegin(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING); - struct.options = new java.util.HashMap(2*_map92.size); - @org.apache.thrift.annotation.Nullable java.lang.String _key93; - @org.apache.thrift.annotation.Nullable java.lang.String _val94; - for (int _i95 = 0; _i95 < _map92.size; ++_i95) + org.apache.thrift.protocol.TMap _map84 = iprot.readMapBegin(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING); + struct.options = new java.util.HashMap(2*_map84.size); + @org.apache.thrift.annotation.Nullable java.lang.String _key85; + @org.apache.thrift.annotation.Nullable java.lang.String _val86; + for (int _i87 = 0; _i87 < _map84.size; ++_i87) { - _key93 = iprot.readString(); - _val94 = iprot.readString(); - struct.options.put(_key93, _val94); + _key85 = iprot.readString(); + _val86 = iprot.readString(); + struct.options.put(_key85, _val86); } } struct.setOptionsIsSet(true); diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/manager/thrift/ManagerClientService.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/manager/thrift/ManagerClientService.java index 9e845fbdcd3..db2078f17ee 100644 --- a/core/src/main/thrift-gen-java/org/apache/accumulo/core/manager/thrift/ManagerClientService.java +++ b/core/src/main/thrift-gen-java/org/apache/accumulo/core/manager/thrift/ManagerClientService.java @@ -35618,13 +35618,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getActiveTservers_r case 0: // SUCCESS if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { { - org.apache.thrift.protocol.TList _list96 = iprot.readListBegin(); - struct.success = new java.util.ArrayList(_list96.size); - @org.apache.thrift.annotation.Nullable java.lang.String _elem97; - for (int _i98 = 0; _i98 < _list96.size; ++_i98) + org.apache.thrift.protocol.TList _list88 = iprot.readListBegin(); + struct.success = new java.util.ArrayList(_list88.size); + @org.apache.thrift.annotation.Nullable java.lang.String _elem89; + for (int _i90 = 0; _i90 < _list88.size; ++_i90) { - _elem97 = iprot.readString(); - struct.success.add(_elem97); + _elem89 = iprot.readString(); + struct.success.add(_elem89); } iprot.readListEnd(); } @@ -35671,9 +35671,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getActiveTservers_ oprot.writeFieldBegin(SUCCESS_FIELD_DESC); { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.success.size())); - for (java.lang.String _iter99 : struct.success) + for (java.lang.String _iter91 : struct.success) { - oprot.writeString(_iter99); + oprot.writeString(_iter91); } oprot.writeListEnd(); } @@ -35721,9 +35721,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getActiveTservers_r if (struct.isSetSuccess()) { { oprot.writeI32(struct.success.size()); - for (java.lang.String _iter100 : struct.success) + for (java.lang.String _iter92 : struct.success) { - oprot.writeString(_iter100); + oprot.writeString(_iter92); } } } @@ -35741,13 +35741,13 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getActiveTservers_re java.util.BitSet incoming = iprot.readBitSet(3); if (incoming.get(0)) { { - org.apache.thrift.protocol.TList _list101 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRING); - struct.success = new java.util.ArrayList(_list101.size); - @org.apache.thrift.annotation.Nullable java.lang.String _elem102; - for (int _i103 = 0; _i103 < _list101.size; ++_i103) + org.apache.thrift.protocol.TList _list93 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRING); + struct.success = new java.util.ArrayList(_list93.size); + @org.apache.thrift.annotation.Nullable java.lang.String _elem94; + for (int _i95 = 0; _i95 < _list93.size; ++_i95) { - _elem102 = iprot.readString(); - struct.success.add(_elem102); + _elem94 = iprot.readString(); + struct.success.add(_elem94); } } struct.setSuccessIsSet(true); @@ -37579,14 +37579,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, requestTabletHostin case 4: // EXTENTS if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { { - org.apache.thrift.protocol.TList _list104 = iprot.readListBegin(); - struct.extents = new java.util.ArrayList(_list104.size); - @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _elem105; - for (int _i106 = 0; _i106 < _list104.size; ++_i106) + org.apache.thrift.protocol.TList _list96 = iprot.readListBegin(); + struct.extents = new java.util.ArrayList(_list96.size); + @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _elem97; + for (int _i98 = 0; _i98 < _list96.size; ++_i98) { - _elem105 = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(); - _elem105.read(iprot); - struct.extents.add(_elem105); + _elem97 = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(); + _elem97.read(iprot); + struct.extents.add(_elem97); } iprot.readListEnd(); } @@ -37630,9 +37630,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, requestTabletHosti oprot.writeFieldBegin(EXTENTS_FIELD_DESC); { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.extents.size())); - for (org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _iter107 : struct.extents) + for (org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _iter99 : struct.extents) { - _iter107.write(oprot); + _iter99.write(oprot); } oprot.writeListEnd(); } @@ -37682,9 +37682,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, requestTabletHostin if (struct.isSetExtents()) { { oprot.writeI32(struct.extents.size()); - for (org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _iter108 : struct.extents) + for (org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _iter100 : struct.extents) { - _iter108.write(oprot); + _iter100.write(oprot); } } } @@ -37710,14 +37710,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, requestTabletHosting } if (incoming.get(3)) { { - org.apache.thrift.protocol.TList _list109 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT); - struct.extents = new java.util.ArrayList(_list109.size); - @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _elem110; - for (int _i111 = 0; _i111 < _list109.size; ++_i111) + org.apache.thrift.protocol.TList _list101 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT); + struct.extents = new java.util.ArrayList(_list101.size); + @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _elem102; + for (int _i103 = 0; _i103 < _list101.size; ++_i103) { - _elem110 = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(); - _elem110.read(iprot); - struct.extents.add(_elem110); + _elem102 = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(); + _elem102.read(iprot); + struct.extents.add(_elem102); } } struct.setExtentsIsSet(true); @@ -38829,17 +38829,17 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, updateTabletMergeab case 4: // SPLITS if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { { - org.apache.thrift.protocol.TMap _map112 = iprot.readMapBegin(); - struct.splits = new java.util.HashMap(2*_map112.size); - @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _key113; - @org.apache.thrift.annotation.Nullable TTabletMergeability _val114; - for (int _i115 = 0; _i115 < _map112.size; ++_i115) + org.apache.thrift.protocol.TMap _map104 = iprot.readMapBegin(); + struct.splits = new java.util.HashMap(2*_map104.size); + @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _key105; + @org.apache.thrift.annotation.Nullable TTabletMergeability _val106; + for (int _i107 = 0; _i107 < _map104.size; ++_i107) { - _key113 = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(); - _key113.read(iprot); - _val114 = new TTabletMergeability(); - _val114.read(iprot); - struct.splits.put(_key113, _val114); + _key105 = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(); + _key105.read(iprot); + _val106 = new TTabletMergeability(); + _val106.read(iprot); + struct.splits.put(_key105, _val106); } iprot.readMapEnd(); } @@ -38883,10 +38883,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, updateTabletMergea oprot.writeFieldBegin(SPLITS_FIELD_DESC); { oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRUCT, org.apache.thrift.protocol.TType.STRUCT, struct.splits.size())); - for (java.util.Map.Entry _iter116 : struct.splits.entrySet()) + for (java.util.Map.Entry _iter108 : struct.splits.entrySet()) { - _iter116.getKey().write(oprot); - _iter116.getValue().write(oprot); + _iter108.getKey().write(oprot); + _iter108.getValue().write(oprot); } oprot.writeMapEnd(); } @@ -38936,10 +38936,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, updateTabletMergeab if (struct.isSetSplits()) { { oprot.writeI32(struct.splits.size()); - for (java.util.Map.Entry _iter117 : struct.splits.entrySet()) + for (java.util.Map.Entry _iter109 : struct.splits.entrySet()) { - _iter117.getKey().write(oprot); - _iter117.getValue().write(oprot); + _iter109.getKey().write(oprot); + _iter109.getValue().write(oprot); } } } @@ -38965,17 +38965,17 @@ public void read(org.apache.thrift.protocol.TProtocol prot, updateTabletMergeabi } if (incoming.get(3)) { { - org.apache.thrift.protocol.TMap _map118 = iprot.readMapBegin(org.apache.thrift.protocol.TType.STRUCT, org.apache.thrift.protocol.TType.STRUCT); - struct.splits = new java.util.HashMap(2*_map118.size); - @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _key119; - @org.apache.thrift.annotation.Nullable TTabletMergeability _val120; - for (int _i121 = 0; _i121 < _map118.size; ++_i121) + org.apache.thrift.protocol.TMap _map110 = iprot.readMapBegin(org.apache.thrift.protocol.TType.STRUCT, org.apache.thrift.protocol.TType.STRUCT); + struct.splits = new java.util.HashMap(2*_map110.size); + @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _key111; + @org.apache.thrift.annotation.Nullable TTabletMergeability _val112; + for (int _i113 = 0; _i113 < _map110.size; ++_i113) { - _key119 = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(); - _key119.read(iprot); - _val120 = new TTabletMergeability(); - _val120.read(iprot); - struct.splits.put(_key119, _val120); + _key111 = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(); + _key111.read(iprot); + _val112 = new TTabletMergeability(); + _val112.read(iprot); + struct.splits.put(_key111, _val112); } } struct.setSplitsIsSet(true); @@ -39481,14 +39481,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, updateTabletMergeab case 0: // SUCCESS if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { { - org.apache.thrift.protocol.TList _list122 = iprot.readListBegin(); - struct.success = new java.util.ArrayList(_list122.size); - @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _elem123; - for (int _i124 = 0; _i124 < _list122.size; ++_i124) + org.apache.thrift.protocol.TList _list114 = iprot.readListBegin(); + struct.success = new java.util.ArrayList(_list114.size); + @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _elem115; + for (int _i116 = 0; _i116 < _list114.size; ++_i116) { - _elem123 = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(); - _elem123.read(iprot); - struct.success.add(_elem123); + _elem115 = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(); + _elem115.read(iprot); + struct.success.add(_elem115); } iprot.readListEnd(); } @@ -39535,9 +39535,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, updateTabletMergea oprot.writeFieldBegin(SUCCESS_FIELD_DESC); { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); - for (org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _iter125 : struct.success) + for (org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _iter117 : struct.success) { - _iter125.write(oprot); + _iter117.write(oprot); } oprot.writeListEnd(); } @@ -39585,9 +39585,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, updateTabletMergeab if (struct.isSetSuccess()) { { oprot.writeI32(struct.success.size()); - for (org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _iter126 : struct.success) + for (org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _iter118 : struct.success) { - _iter126.write(oprot); + _iter118.write(oprot); } } } @@ -39605,14 +39605,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, updateTabletMergeabi java.util.BitSet incoming = iprot.readBitSet(3); if (incoming.get(0)) { { - org.apache.thrift.protocol.TList _list127 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT); - struct.success = new java.util.ArrayList(_list127.size); - @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _elem128; - for (int _i129 = 0; _i129 < _list127.size; ++_i129) + org.apache.thrift.protocol.TList _list119 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT); + struct.success = new java.util.ArrayList(_list119.size); + @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _elem120; + for (int _i121 = 0; _i121 < _list119.size; ++_i121) { - _elem128 = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(); - _elem128.read(iprot); - struct.success.add(_elem128); + _elem120 = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(); + _elem120.read(iprot); + struct.success.add(_elem120); } } struct.setSuccessIsSet(true); diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/manager/thrift/ManagerMonitorInfo.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/manager/thrift/ManagerMonitorInfo.java index 66a43f5c6b9..8a028db0cee 100644 --- a/core/src/main/thrift-gen-java/org/apache/accumulo/core/manager/thrift/ManagerMonitorInfo.java +++ b/core/src/main/thrift-gen-java/org/apache/accumulo/core/manager/thrift/ManagerMonitorInfo.java @@ -36,7 +36,6 @@ public class ManagerMonitorInfo implements org.apache.thrift.TBase serversShuttingDown; // required public @org.apache.thrift.annotation.Nullable java.util.List deadTabletServers; // required - public @org.apache.thrift.annotation.Nullable java.util.List bulkImports; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { @@ -76,8 +74,7 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum { GOAL_STATE((short)5, "goalState"), UNASSIGNED_TABLETS((short)6, "unassignedTablets"), SERVERS_SHUTTING_DOWN((short)7, "serversShuttingDown"), - DEAD_TABLET_SERVERS((short)8, "deadTabletServers"), - BULK_IMPORTS((short)9, "bulkImports"); + DEAD_TABLET_SERVERS((short)8, "deadTabletServers"); private static final java.util.Map byName = new java.util.HashMap(); @@ -109,8 +106,6 @@ public static _Fields findByThriftId(int fieldId) { return SERVERS_SHUTTING_DOWN; case 8: // DEAD_TABLET_SERVERS return DEAD_TABLET_SERVERS; - case 9: // BULK_IMPORTS - return BULK_IMPORTS; default: return null; } @@ -182,9 +177,6 @@ public java.lang.String getFieldName() { tmpMap.put(_Fields.DEAD_TABLET_SERVERS, new org.apache.thrift.meta_data.FieldMetaData("deadTabletServers", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, DeadServer.class)))); - tmpMap.put(_Fields.BULK_IMPORTS, new org.apache.thrift.meta_data.FieldMetaData("bulkImports", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, BulkImportStatus.class)))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(ManagerMonitorInfo.class, metaDataMap); } @@ -200,8 +192,7 @@ public ManagerMonitorInfo( ManagerGoalState goalState, int unassignedTablets, java.util.Set serversShuttingDown, - java.util.List deadTabletServers, - java.util.List bulkImports) + java.util.List deadTabletServers) { this(); this.tableMap = tableMap; @@ -213,7 +204,6 @@ public ManagerMonitorInfo( setUnassignedTabletsIsSet(true); this.serversShuttingDown = serversShuttingDown; this.deadTabletServers = deadTabletServers; - this.bulkImports = bulkImports; } /** @@ -265,13 +255,6 @@ public ManagerMonitorInfo(ManagerMonitorInfo other) { } this.deadTabletServers = __this__deadTabletServers; } - if (other.isSetBulkImports()) { - java.util.List __this__bulkImports = new java.util.ArrayList(other.bulkImports.size()); - for (BulkImportStatus other_element : other.bulkImports) { - __this__bulkImports.add(new BulkImportStatus(other_element)); - } - this.bulkImports = __this__bulkImports; - } } @Override @@ -290,7 +273,6 @@ public void clear() { this.unassignedTablets = 0; this.serversShuttingDown = null; this.deadTabletServers = null; - this.bulkImports = null; } public int getTableMapSize() { @@ -577,47 +559,6 @@ public void setDeadTabletServersIsSet(boolean value) { } } - public int getBulkImportsSize() { - return (this.bulkImports == null) ? 0 : this.bulkImports.size(); - } - - @org.apache.thrift.annotation.Nullable - public java.util.Iterator getBulkImportsIterator() { - return (this.bulkImports == null) ? null : this.bulkImports.iterator(); - } - - public void addToBulkImports(BulkImportStatus elem) { - if (this.bulkImports == null) { - this.bulkImports = new java.util.ArrayList(); - } - this.bulkImports.add(elem); - } - - @org.apache.thrift.annotation.Nullable - public java.util.List getBulkImports() { - return this.bulkImports; - } - - public ManagerMonitorInfo setBulkImports(@org.apache.thrift.annotation.Nullable java.util.List bulkImports) { - this.bulkImports = bulkImports; - return this; - } - - public void unsetBulkImports() { - this.bulkImports = null; - } - - /** Returns true if field bulkImports is set (has been assigned a value) and false otherwise */ - public boolean isSetBulkImports() { - return this.bulkImports != null; - } - - public void setBulkImportsIsSet(boolean value) { - if (!value) { - this.bulkImports = null; - } - } - @Override public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { switch (field) { @@ -685,14 +626,6 @@ public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable } break; - case BULK_IMPORTS: - if (value == null) { - unsetBulkImports(); - } else { - setBulkImports((java.util.List)value); - } - break; - } } @@ -724,9 +657,6 @@ public java.lang.Object getFieldValue(_Fields field) { case DEAD_TABLET_SERVERS: return getDeadTabletServers(); - case BULK_IMPORTS: - return getBulkImports(); - } throw new java.lang.IllegalStateException(); } @@ -755,8 +685,6 @@ public boolean isSet(_Fields field) { return isSetServersShuttingDown(); case DEAD_TABLET_SERVERS: return isSetDeadTabletServers(); - case BULK_IMPORTS: - return isSetBulkImports(); } throw new java.lang.IllegalStateException(); } @@ -846,15 +774,6 @@ public boolean equals(ManagerMonitorInfo that) { return false; } - boolean this_present_bulkImports = true && this.isSetBulkImports(); - boolean that_present_bulkImports = true && that.isSetBulkImports(); - if (this_present_bulkImports || that_present_bulkImports) { - if (!(this_present_bulkImports && that_present_bulkImports)) - return false; - if (!this.bulkImports.equals(that.bulkImports)) - return false; - } - return true; } @@ -892,10 +811,6 @@ public int hashCode() { if (isSetDeadTabletServers()) hashCode = hashCode * 8191 + deadTabletServers.hashCode(); - hashCode = hashCode * 8191 + ((isSetBulkImports()) ? 131071 : 524287); - if (isSetBulkImports()) - hashCode = hashCode * 8191 + bulkImports.hashCode(); - return hashCode; } @@ -987,16 +902,6 @@ public int compareTo(ManagerMonitorInfo other) { return lastComparison; } } - lastComparison = java.lang.Boolean.compare(isSetBulkImports(), other.isSetBulkImports()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetBulkImports()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.bulkImports, other.bulkImports); - if (lastComparison != 0) { - return lastComparison; - } - } return 0; } @@ -1080,14 +985,6 @@ public java.lang.String toString() { sb.append(this.deadTabletServers); } first = false; - if (!first) sb.append(", "); - sb.append("bulkImports:"); - if (this.bulkImports == null) { - sb.append("null"); - } else { - sb.append(this.bulkImports); - } - first = false; sb.append(")"); return sb.toString(); } @@ -1256,25 +1153,6 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, ManagerMonitorInfo org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 9: // BULK_IMPORTS - if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { - { - org.apache.thrift.protocol.TList _list43 = iprot.readListBegin(); - struct.bulkImports = new java.util.ArrayList(_list43.size); - @org.apache.thrift.annotation.Nullable BulkImportStatus _elem44; - for (int _i45 = 0; _i45 < _list43.size; ++_i45) - { - _elem44 = new BulkImportStatus(); - _elem44.read(iprot); - struct.bulkImports.add(_elem44); - } - iprot.readListEnd(); - } - struct.setBulkImportsIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -1295,10 +1173,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, ManagerMonitorInfo oprot.writeFieldBegin(TABLE_MAP_FIELD_DESC); { oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, struct.tableMap.size())); - for (java.util.Map.Entry _iter46 : struct.tableMap.entrySet()) + for (java.util.Map.Entry _iter43 : struct.tableMap.entrySet()) { - oprot.writeString(_iter46.getKey()); - _iter46.getValue().write(oprot); + oprot.writeString(_iter43.getKey()); + _iter43.getValue().write(oprot); } oprot.writeMapEnd(); } @@ -1308,9 +1186,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, ManagerMonitorInfo oprot.writeFieldBegin(T_SERVER_INFO_FIELD_DESC); { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.tServerInfo.size())); - for (TabletServerStatus _iter47 : struct.tServerInfo) + for (TabletServerStatus _iter44 : struct.tServerInfo) { - _iter47.write(oprot); + _iter44.write(oprot); } oprot.writeListEnd(); } @@ -1320,10 +1198,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, ManagerMonitorInfo oprot.writeFieldBegin(BAD_TSERVERS_FIELD_DESC); { oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.BYTE, struct.badTServers.size())); - for (java.util.Map.Entry _iter48 : struct.badTServers.entrySet()) + for (java.util.Map.Entry _iter45 : struct.badTServers.entrySet()) { - oprot.writeString(_iter48.getKey()); - oprot.writeByte(_iter48.getValue()); + oprot.writeString(_iter45.getKey()); + oprot.writeByte(_iter45.getValue()); } oprot.writeMapEnd(); } @@ -1346,9 +1224,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, ManagerMonitorInfo oprot.writeFieldBegin(SERVERS_SHUTTING_DOWN_FIELD_DESC); { oprot.writeSetBegin(new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.STRING, struct.serversShuttingDown.size())); - for (java.lang.String _iter49 : struct.serversShuttingDown) + for (java.lang.String _iter46 : struct.serversShuttingDown) { - oprot.writeString(_iter49); + oprot.writeString(_iter46); } oprot.writeSetEnd(); } @@ -1358,21 +1236,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, ManagerMonitorInfo oprot.writeFieldBegin(DEAD_TABLET_SERVERS_FIELD_DESC); { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.deadTabletServers.size())); - for (DeadServer _iter50 : struct.deadTabletServers) + for (DeadServer _iter47 : struct.deadTabletServers) { - _iter50.write(oprot); - } - oprot.writeListEnd(); - } - oprot.writeFieldEnd(); - } - if (struct.bulkImports != null) { - oprot.writeFieldBegin(BULK_IMPORTS_FIELD_DESC); - { - oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.bulkImports.size())); - for (BulkImportStatus _iter51 : struct.bulkImports) - { - _iter51.write(oprot); + _iter47.write(oprot); } oprot.writeListEnd(); } @@ -1421,36 +1287,33 @@ public void write(org.apache.thrift.protocol.TProtocol prot, ManagerMonitorInfo if (struct.isSetDeadTabletServers()) { optionals.set(7); } - if (struct.isSetBulkImports()) { - optionals.set(8); - } - oprot.writeBitSet(optionals, 9); + oprot.writeBitSet(optionals, 8); if (struct.isSetTableMap()) { { oprot.writeI32(struct.tableMap.size()); - for (java.util.Map.Entry _iter52 : struct.tableMap.entrySet()) + for (java.util.Map.Entry _iter48 : struct.tableMap.entrySet()) { - oprot.writeString(_iter52.getKey()); - _iter52.getValue().write(oprot); + oprot.writeString(_iter48.getKey()); + _iter48.getValue().write(oprot); } } } if (struct.isSetTServerInfo()) { { oprot.writeI32(struct.tServerInfo.size()); - for (TabletServerStatus _iter53 : struct.tServerInfo) + for (TabletServerStatus _iter49 : struct.tServerInfo) { - _iter53.write(oprot); + _iter49.write(oprot); } } } if (struct.isSetBadTServers()) { { oprot.writeI32(struct.badTServers.size()); - for (java.util.Map.Entry _iter54 : struct.badTServers.entrySet()) + for (java.util.Map.Entry _iter50 : struct.badTServers.entrySet()) { - oprot.writeString(_iter54.getKey()); - oprot.writeByte(_iter54.getValue()); + oprot.writeString(_iter50.getKey()); + oprot.writeByte(_iter50.getValue()); } } } @@ -1466,27 +1329,18 @@ public void write(org.apache.thrift.protocol.TProtocol prot, ManagerMonitorInfo if (struct.isSetServersShuttingDown()) { { oprot.writeI32(struct.serversShuttingDown.size()); - for (java.lang.String _iter55 : struct.serversShuttingDown) + for (java.lang.String _iter51 : struct.serversShuttingDown) { - oprot.writeString(_iter55); + oprot.writeString(_iter51); } } } if (struct.isSetDeadTabletServers()) { { oprot.writeI32(struct.deadTabletServers.size()); - for (DeadServer _iter56 : struct.deadTabletServers) + for (DeadServer _iter52 : struct.deadTabletServers) { - _iter56.write(oprot); - } - } - } - if (struct.isSetBulkImports()) { - { - oprot.writeI32(struct.bulkImports.size()); - for (BulkImportStatus _iter57 : struct.bulkImports) - { - _iter57.write(oprot); + _iter52.write(oprot); } } } @@ -1495,48 +1349,48 @@ public void write(org.apache.thrift.protocol.TProtocol prot, ManagerMonitorInfo @Override public void read(org.apache.thrift.protocol.TProtocol prot, ManagerMonitorInfo struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(9); + java.util.BitSet incoming = iprot.readBitSet(8); if (incoming.get(0)) { { - org.apache.thrift.protocol.TMap _map58 = iprot.readMapBegin(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT); - struct.tableMap = new java.util.HashMap(2*_map58.size); - @org.apache.thrift.annotation.Nullable java.lang.String _key59; - @org.apache.thrift.annotation.Nullable TableInfo _val60; - for (int _i61 = 0; _i61 < _map58.size; ++_i61) + org.apache.thrift.protocol.TMap _map53 = iprot.readMapBegin(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT); + struct.tableMap = new java.util.HashMap(2*_map53.size); + @org.apache.thrift.annotation.Nullable java.lang.String _key54; + @org.apache.thrift.annotation.Nullable TableInfo _val55; + for (int _i56 = 0; _i56 < _map53.size; ++_i56) { - _key59 = iprot.readString(); - _val60 = new TableInfo(); - _val60.read(iprot); - struct.tableMap.put(_key59, _val60); + _key54 = iprot.readString(); + _val55 = new TableInfo(); + _val55.read(iprot); + struct.tableMap.put(_key54, _val55); } } struct.setTableMapIsSet(true); } if (incoming.get(1)) { { - org.apache.thrift.protocol.TList _list62 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT); - struct.tServerInfo = new java.util.ArrayList(_list62.size); - @org.apache.thrift.annotation.Nullable TabletServerStatus _elem63; - for (int _i64 = 0; _i64 < _list62.size; ++_i64) + org.apache.thrift.protocol.TList _list57 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT); + struct.tServerInfo = new java.util.ArrayList(_list57.size); + @org.apache.thrift.annotation.Nullable TabletServerStatus _elem58; + for (int _i59 = 0; _i59 < _list57.size; ++_i59) { - _elem63 = new TabletServerStatus(); - _elem63.read(iprot); - struct.tServerInfo.add(_elem63); + _elem58 = new TabletServerStatus(); + _elem58.read(iprot); + struct.tServerInfo.add(_elem58); } } struct.setTServerInfoIsSet(true); } if (incoming.get(2)) { { - org.apache.thrift.protocol.TMap _map65 = iprot.readMapBegin(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.BYTE); - struct.badTServers = new java.util.HashMap(2*_map65.size); - @org.apache.thrift.annotation.Nullable java.lang.String _key66; - byte _val67; - for (int _i68 = 0; _i68 < _map65.size; ++_i68) + org.apache.thrift.protocol.TMap _map60 = iprot.readMapBegin(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.BYTE); + struct.badTServers = new java.util.HashMap(2*_map60.size); + @org.apache.thrift.annotation.Nullable java.lang.String _key61; + byte _val62; + for (int _i63 = 0; _i63 < _map60.size; ++_i63) { - _key66 = iprot.readString(); - _val67 = iprot.readByte(); - struct.badTServers.put(_key66, _val67); + _key61 = iprot.readString(); + _val62 = iprot.readByte(); + struct.badTServers.put(_key61, _val62); } } struct.setBadTServersIsSet(true); @@ -1555,45 +1409,31 @@ public void read(org.apache.thrift.protocol.TProtocol prot, ManagerMonitorInfo s } if (incoming.get(6)) { { - org.apache.thrift.protocol.TSet _set69 = iprot.readSetBegin(org.apache.thrift.protocol.TType.STRING); - struct.serversShuttingDown = new java.util.HashSet(2*_set69.size); - @org.apache.thrift.annotation.Nullable java.lang.String _elem70; - for (int _i71 = 0; _i71 < _set69.size; ++_i71) + org.apache.thrift.protocol.TSet _set64 = iprot.readSetBegin(org.apache.thrift.protocol.TType.STRING); + struct.serversShuttingDown = new java.util.HashSet(2*_set64.size); + @org.apache.thrift.annotation.Nullable java.lang.String _elem65; + for (int _i66 = 0; _i66 < _set64.size; ++_i66) { - _elem70 = iprot.readString(); - struct.serversShuttingDown.add(_elem70); + _elem65 = iprot.readString(); + struct.serversShuttingDown.add(_elem65); } } struct.setServersShuttingDownIsSet(true); } if (incoming.get(7)) { { - org.apache.thrift.protocol.TList _list72 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT); - struct.deadTabletServers = new java.util.ArrayList(_list72.size); - @org.apache.thrift.annotation.Nullable DeadServer _elem73; - for (int _i74 = 0; _i74 < _list72.size; ++_i74) + org.apache.thrift.protocol.TList _list67 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT); + struct.deadTabletServers = new java.util.ArrayList(_list67.size); + @org.apache.thrift.annotation.Nullable DeadServer _elem68; + for (int _i69 = 0; _i69 < _list67.size; ++_i69) { - _elem73 = new DeadServer(); - _elem73.read(iprot); - struct.deadTabletServers.add(_elem73); + _elem68 = new DeadServer(); + _elem68.read(iprot); + struct.deadTabletServers.add(_elem68); } } struct.setDeadTabletServersIsSet(true); } - if (incoming.get(8)) { - { - org.apache.thrift.protocol.TList _list75 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT); - struct.bulkImports = new java.util.ArrayList(_list75.size); - @org.apache.thrift.annotation.Nullable BulkImportStatus _elem76; - for (int _i77 = 0; _i77 < _list75.size; ++_i77) - { - _elem76 = new BulkImportStatus(); - _elem76.read(iprot); - struct.bulkImports.add(_elem76); - } - } - struct.setBulkImportsIsSet(true); - } } } diff --git a/core/src/main/thrift/manager.thrift b/core/src/main/thrift/manager.thrift index 436d365e979..c9bbe45f409 100644 --- a/core/src/main/thrift/manager.thrift +++ b/core/src/main/thrift/manager.thrift @@ -113,28 +113,6 @@ struct RecoveryStatus { 6:double progress } -enum BulkImportState { - INITIAL - // manager moves the files into the accumulo area - MOVING - // tserver examines the index of the file - PROCESSING - // tserver assigns the file to tablets - ASSIGNING - // tserver incorporates file into tablet - LOADING - // manager moves error files into the error directory - COPY_FILES - // flags and locks removed - CLEANUP -} - -struct BulkImportStatus { - 1:i64 startTime - 2:string filename - 3:BulkImportState state -} - struct TabletServerStatus { 1:map tableMap 2:i64 lastContact @@ -162,7 +140,6 @@ struct ManagerMonitorInfo { 6:i32 unassignedTablets 7:set serversShuttingDown 8:list deadTabletServers - 9:list bulkImports } enum TFateInstanceType { diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ServerBulkImportStatus.java b/server/base/src/main/java/org/apache/accumulo/server/util/ServerBulkImportStatus.java deleted file mode 100644 index 1ab394b29d3..00000000000 --- a/server/base/src/main/java/org/apache/accumulo/server/util/ServerBulkImportStatus.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.server.util; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -import org.apache.accumulo.core.manager.thrift.BulkImportState; -import org.apache.accumulo.core.manager.thrift.BulkImportStatus; - -// A little class to hold bulk import status information in the Manager -// and two places in the tablet server. -public class ServerBulkImportStatus { - private final ConcurrentMap status = new ConcurrentHashMap<>(); - - public List getBulkLoadStatus() { - return new ArrayList<>(status.values()); - } - - public void updateBulkImportStatus(List files, BulkImportState state) { - for (String file : files) { - status.compute(file, (key, currentStatus) -> { - if (currentStatus == null) { - return new BulkImportStatus(System.currentTimeMillis(), file, state); - } - currentStatus.state = state; - return currentStatus; - }); - } - } - - public void removeBulkImportStatus(List files) { - status.keySet().removeAll(files); - } - -} diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/bulkCommand/BulkFateOperation.java b/server/base/src/main/java/org/apache/accumulo/server/util/bulkCommand/BulkFateOperation.java new file mode 100644 index 00000000000..28edbf65ddc --- /dev/null +++ b/server/base/src/main/java/org/apache/accumulo/server/util/bulkCommand/BulkFateOperation.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.util.bulkCommand; + +import java.time.Instant; + +import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.server.util.bulkCommand.ListBulk.BulkState; + +public interface BulkFateOperation { + TableId getTableId(); + + String getSourceDir(); + + String getDestDir(); + + Instant getCreationTime(); + + BulkState getState(); +} diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/bulkCommand/ListBulk.java b/server/base/src/main/java/org/apache/accumulo/server/util/bulkCommand/ListBulk.java new file mode 100644 index 00000000000..8621a0ab7be --- /dev/null +++ b/server/base/src/main/java/org/apache/accumulo/server/util/bulkCommand/ListBulk.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.util.bulkCommand; + +import java.time.Duration; +import java.time.Instant; +import java.util.EnumSet; +import java.util.function.Consumer; +import java.util.stream.Stream; + +import org.apache.accumulo.core.cli.ServerOpts; +import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.fate.Fate; +import org.apache.accumulo.core.fate.FateId; +import org.apache.accumulo.core.fate.ReadOnlyFateStore; +import org.apache.accumulo.core.fate.ReadOnlyFateStore.FateIdStatus; +import org.apache.accumulo.core.fate.TraceRepo; +import org.apache.accumulo.core.fate.user.UserFateStore; +import org.apache.accumulo.core.metadata.SystemTables; +import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.util.ServerKeywordExecutable; +import org.apache.accumulo.start.spi.CommandGroup; +import org.apache.accumulo.start.spi.CommandGroups; + +import com.beust.jcommander.JCommander; + +public class ListBulk extends ServerKeywordExecutable { + + public ListBulk() { + super(new ServerOpts()); + } + + @Override + public String keyword() { + return "list-bulk"; + } + + @Override + public CommandGroup commandGroup() { + return CommandGroups.INSTANCE; + } + + @Override + public String description() { + return "List current bulk import operations"; + } + + @Override + public void execute(JCommander cl, ServerOpts options) throws Exception { + ServerContext context = getServerContext(); + System.out.printf(" %-46s | %-8s | %-10s | %-7s | %-20s | %-20s\n", "FATE ID", "TABLE ID", + "STATE", "AGE", "SOURCE DIR", "DESTINATION DIR"); + var now = Instant.now(); + list(context, bs -> { + var elapsed = Duration.between(bs.lastUpdate, now); + System.out.printf(" %46s | %8s | %10s | %d:%02d:%02d | %20s | %20s\n", bs.fateId, bs.tableId, + bs.state, elapsed.toHours(), elapsed.toMinutesPart(), elapsed.toSecondsPart(), + bs.sourceDir, bs.destDir); + }); + } + + public enum BulkState { + PREPARING, MOVING, LOADING, REFRESHING, CLEANING + } + + public record BulkStatus(FateId fateId, TableId tableId, String sourceDir, String destDir, + Instant lastUpdate, BulkState state) { + } + + /** + * Gathers summary information about all running bulk import operations. + */ + public static void list(ServerContext context, Consumer statusConsumer) { + var fateStore = new UserFateStore<>(context, SystemTables.FATE.tableName(), null, null); + + try (var fateStream = fateStore.list( + EnumSet.of(ReadOnlyFateStore.TStatus.SUBMITTED, ReadOnlyFateStore.TStatus.IN_PROGRESS))) { + fateStream + .filter(fis -> fis.getFateOperation() + .map(op -> op == Fate.FateOperation.TABLE_BULK_IMPORT2).orElse(false)) + .map(FateIdStatus::getFateId).flatMap(fateId -> getStatus(fateStore, fateId)) + .forEach(statusConsumer); + } + } + + private static Stream getStatus(UserFateStore fateStore, FateId fateId) { + var txStore = fateStore.read(fateId); + var top = txStore.top(); + if (top == null) { + return Stream.of(); + } + + BulkFateOperation bfo; + if (top instanceof TraceRepo) { + var wrapped = ((TraceRepo) top).getWrapped(); + if (wrapped instanceof BulkFateOperation) { + bfo = (BulkFateOperation) wrapped; + } else { + throw new IllegalStateException( + "Unknown fate operation class type " + wrapped.getClass().getName()); + } + } else if (top instanceof BulkFateOperation) { + bfo = (BulkFateOperation) top; + } else { + throw new IllegalStateException( + "Unknown fate operation class type " + top.getClass().getName()); + } + + return Stream.of(new ListBulk.BulkStatus(fateId, bfo.getTableId(), bfo.getSourceDir(), + bfo.getDestDir(), bfo.getCreationTime(), bfo.getState())); + } + +} diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java b/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java index 6c31e9174b7..f48ef33e19d 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java @@ -73,9 +73,9 @@ import org.apache.accumulo.core.fate.FateId; import org.apache.accumulo.core.fate.FateInstanceType; import org.apache.accumulo.core.fate.ReadOnlyFateStore.TStatus; +import org.apache.accumulo.core.fate.TraceRepo; import org.apache.accumulo.core.iteratorsImpl.IteratorConfigUtil; import org.apache.accumulo.core.manager.state.tables.TableState; -import org.apache.accumulo.core.manager.thrift.BulkImportState; import org.apache.accumulo.core.manager.thrift.FateService; import org.apache.accumulo.core.manager.thrift.TFateId; import org.apache.accumulo.core.manager.thrift.TFateInstanceType; @@ -87,7 +87,6 @@ import org.apache.accumulo.core.util.tables.TableNameUtil; import org.apache.accumulo.core.volume.Volume; import org.apache.accumulo.manager.tableOps.ChangeTableState; -import org.apache.accumulo.manager.tableOps.TraceRepo; import org.apache.accumulo.manager.tableOps.availability.LockTable; import org.apache.accumulo.manager.tableOps.bulkVer2.ComputeBulkRange; import org.apache.accumulo.manager.tableOps.clone.CloneTable; @@ -673,8 +672,6 @@ public void executeFateOperation(TInfo tinfo, TCredentials c, TFateId opid, TFat throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED); } - manager.updateBulkImportStatus(dir, BulkImportState.INITIAL); - goalMessage += "Bulk import (v2) " + dir + " to " + tableName + "(" + tableId + ")"; manager.fate(type).seedTransaction(op, fateId, new TraceRepo<>(new ComputeBulkRange(tableId, dir, setTime)), autoCleanup, goalMessage); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java index ec6df2d071d..06f227967ef 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java @@ -71,6 +71,7 @@ import org.apache.accumulo.core.fate.FateId; import org.apache.accumulo.core.fate.FateInstanceType; import org.apache.accumulo.core.fate.FateStore; +import org.apache.accumulo.core.fate.TraceRepo; import org.apache.accumulo.core.fate.user.UserFateStore; import org.apache.accumulo.core.fate.zookeeper.MetaFateStore; import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; @@ -87,7 +88,6 @@ import org.apache.accumulo.core.lock.ServiceLockSupport.HAServiceLockWatcher; import org.apache.accumulo.core.logging.ConditionalLogger.DeduplicatingLogger; import org.apache.accumulo.core.manager.state.tables.TableState; -import org.apache.accumulo.core.manager.thrift.BulkImportState; import org.apache.accumulo.core.manager.thrift.ManagerGoalState; import org.apache.accumulo.core.manager.thrift.ManagerMonitorInfo; import org.apache.accumulo.core.manager.thrift.ManagerState; @@ -115,7 +115,6 @@ import org.apache.accumulo.manager.split.Splitter; import org.apache.accumulo.manager.state.TableCounts; import org.apache.accumulo.manager.tableOps.FateEnv; -import org.apache.accumulo.manager.tableOps.TraceRepo; import org.apache.accumulo.manager.upgrade.UpgradeCoordinator; import org.apache.accumulo.manager.upgrade.UpgradeCoordinator.UpgradeStatus; import org.apache.accumulo.server.AbstractServer; @@ -134,7 +133,6 @@ import org.apache.accumulo.server.security.delegation.ZooAuthenticationKeyDistributor; import org.apache.accumulo.server.tables.TableManager; import org.apache.accumulo.server.util.ScanServerMetadataEntries; -import org.apache.accumulo.server.util.ServerBulkImportStatus; import org.apache.accumulo.server.util.TableInfoUtil; import org.apache.thrift.TException; import org.apache.thrift.transport.TTransportException; @@ -260,8 +258,6 @@ void setTserverStatus(LiveTServersSnapshot snapshot, } } - final ServerBulkImportStatus bulkImportStatus = new ServerBulkImportStatus(); - private final long timeToCacheRecoveryWalExistence; private ExecutorService tableInformationStatusPool = null; private ThreadPoolExecutor tabletRefreshThreadPool; @@ -1588,7 +1584,6 @@ public ManagerMonitorInfo getManagerMonitorInfo() { } DeadServerList obit = new DeadServerList(getContext()); result.deadTabletServers = obit.getList(); - result.bulkImports = bulkImportStatus.getBulkLoadStatus(); return result; } @@ -1605,16 +1600,6 @@ public Set shutdownServers() { } } - @Override - public void updateBulkImportStatus(String directory, BulkImportState state) { - bulkImportStatus.updateBulkImportStatus(Collections.singletonList(directory), state); - } - - @Override - public void removeBulkImportStatus(String directory) { - bulkImportStatus.removeBulkImportStatus(Collections.singletonList(directory)); - } - /** * Return how long there has been a manager overseeing this cluster. This is an approximately * monotonic clock, which will be approximately consistent between different managers or different diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/ManagerClientServiceHandler.java b/server/manager/src/main/java/org/apache/accumulo/manager/ManagerClientServiceHandler.java index 6f8d284a240..1713dca5921 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/ManagerClientServiceHandler.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/ManagerClientServiceHandler.java @@ -61,6 +61,7 @@ import org.apache.accumulo.core.fate.Fate; import org.apache.accumulo.core.fate.FateId; import org.apache.accumulo.core.fate.FateInstanceType; +import org.apache.accumulo.core.fate.TraceRepo; import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; import org.apache.accumulo.core.manager.state.tables.TableState; import org.apache.accumulo.core.manager.thrift.ManagerClientService; @@ -82,7 +83,6 @@ import org.apache.accumulo.core.securityImpl.thrift.TDelegationTokenConfig; import org.apache.accumulo.core.util.ByteBufferUtil; import org.apache.accumulo.manager.tableOps.FateEnv; -import org.apache.accumulo.manager.tableOps.TraceRepo; import org.apache.accumulo.manager.tserverOps.ShutdownTServer; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.client.ClientServiceHandler; diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/merge/FindMergeableRangeTask.java b/server/manager/src/main/java/org/apache/accumulo/manager/merge/FindMergeableRangeTask.java index e4b39229ef1..a59bee95ef7 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/merge/FindMergeableRangeTask.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/merge/FindMergeableRangeTask.java @@ -41,12 +41,12 @@ import org.apache.accumulo.core.fate.Fate.FateOperation; import org.apache.accumulo.core.fate.FateInstanceType; import org.apache.accumulo.core.fate.FateKey; +import org.apache.accumulo.core.fate.TraceRepo; import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType; import org.apache.accumulo.core.metadata.schema.filters.TabletMetadataFilter; import org.apache.accumulo.core.util.time.SteadyTime; import org.apache.accumulo.manager.Manager; -import org.apache.accumulo.manager.tableOps.TraceRepo; import org.apache.accumulo.manager.tableOps.merge.MergeInfo.Operation; import org.apache.accumulo.manager.tableOps.merge.TableRangeOp; import org.apache.commons.lang3.StringUtils; diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/FateEnv.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/FateEnv.java index e8e0085e3db..f64b3851ae6 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/FateEnv.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/FateEnv.java @@ -22,7 +22,6 @@ import java.util.concurrent.ExecutorService; import org.apache.accumulo.core.lock.ServiceLock; -import org.apache.accumulo.core.manager.thrift.BulkImportState; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.ExternalCompactionId; import org.apache.accumulo.core.util.time.SteadyTime; @@ -45,10 +44,6 @@ public interface FateEnv { VolumeManager getVolumeManager(); - void updateBulkImportStatus(String string, BulkImportState bulkImportState); - - void removeBulkImportStatus(String sourceDir); - ServiceLock getServiceLock(); SteadyTime getSteadyTime(); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/AbstractBulkFateOperation.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/AbstractBulkFateOperation.java new file mode 100644 index 00000000000..a91520ad7dc --- /dev/null +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/AbstractBulkFateOperation.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.manager.tableOps.bulkVer2; + +import java.time.Instant; + +import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.manager.tableOps.AbstractFateOperation; +import org.apache.accumulo.server.util.bulkCommand.BulkFateOperation; + +public abstract class AbstractBulkFateOperation extends AbstractFateOperation + implements BulkFateOperation { + + private static final long serialVersionUID = 1L; + + protected final BulkInfo bulkInfo; + private final Instant creation; + + AbstractBulkFateOperation(BulkInfo bulkInfo) { + this.bulkInfo = bulkInfo; + this.creation = Instant.now(); + } + + @Override + public TableId getTableId() { + return bulkInfo.tableId; + } + + @Override + public String getSourceDir() { + return bulkInfo.sourceDir; + } + + @Override + public String getDestDir() { + return bulkInfo.bulkDir; + } + + @Override + public Instant getCreationTime() { + return creation; + } +} diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/BulkImportMove.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/BulkImportMove.java index be562273ba0..d86b2701925 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/BulkImportMove.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/BulkImportMove.java @@ -29,10 +29,9 @@ import org.apache.accumulo.core.fate.FateId; import org.apache.accumulo.core.fate.Repo; import org.apache.accumulo.core.logging.BulkLogger; -import org.apache.accumulo.core.manager.thrift.BulkImportState; -import org.apache.accumulo.manager.tableOps.AbstractFateOperation; import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.util.bulkCommand.ListBulk.BulkState; import org.apache.hadoop.fs.Path; /** @@ -50,14 +49,12 @@ * about the request. To prevent problems like this, an Arbitrator is used. Before starting any new * request, the tablet server checks the Arbitrator to see if the request is still valid. */ -class BulkImportMove extends AbstractFateOperation { +class BulkImportMove extends AbstractBulkFateOperation { private static final long serialVersionUID = 1L; - private final BulkInfo bulkInfo; - public BulkImportMove(BulkInfo bulkInfo) { - this.bulkInfo = bulkInfo; + super(bulkInfo); } @Override @@ -68,7 +65,6 @@ public Repo call(FateId fateId, FateEnv env) throws Exception { VolumeManager fs = env.getVolumeManager(); try { - env.updateBulkImportStatus(sourceDir.toString(), BulkImportState.MOVING); Map oldToNewNameMap = BulkSerialize.readRenameMap(bulkDir.toString(), fs::open); moveFiles(fateId, sourceDir, bulkDir, env, fs, oldToNewNameMap); @@ -104,4 +100,9 @@ private void moveFiles(FateId fateId, Path sourceDir, Path bulkDir, FateEnv env, ioe.getCause().getMessage()); } } + + @Override + public BulkState getState() { + return BulkState.MOVING; + } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/BulkInfo.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/BulkInfo.java index 3ab809b89d6..47bfc4c432d 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/BulkInfo.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/BulkInfo.java @@ -36,4 +36,12 @@ class BulkInfo implements Serializable { // imported data byte[] firstSplit; byte[] lastSplit; + + static BulkInfo create(TableId tableId, String sourceDir, boolean setTime) { + BulkInfo info = new BulkInfo(); + info.tableId = tableId; + info.sourceDir = sourceDir; + info.setTime = setTime; + return info; + } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java index f22127d962e..b9229bf8f67 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java @@ -32,52 +32,48 @@ import org.apache.accumulo.core.fate.zookeeper.LockRange; import org.apache.accumulo.core.gc.ReferenceFile; import org.apache.accumulo.core.logging.BulkLogger; -import org.apache.accumulo.core.manager.thrift.BulkImportState; import org.apache.accumulo.core.metadata.schema.Ample; import org.apache.accumulo.core.metadata.schema.Ample.ConditionalResult.Status; import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType; import org.apache.accumulo.core.util.Retry; -import org.apache.accumulo.manager.tableOps.AbstractFateOperation; import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.manager.tableOps.Utils; +import org.apache.accumulo.server.util.bulkCommand.ListBulk.BulkState; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class CleanUpBulkImport extends AbstractFateOperation { +public class CleanUpBulkImport extends AbstractBulkFateOperation { private static final long serialVersionUID = 1L; private static final Logger log = LoggerFactory.getLogger(CleanUpBulkImport.class); - private final BulkInfo info; - public CleanUpBulkImport(BulkInfo info) { - this.info = info; + super(info); } @Override public Repo call(FateId fateId, FateEnv env) throws Exception { - env.updateBulkImportStatus(info.sourceDir, BulkImportState.CLEANUP); - log.debug("{} removing the bulkDir processing flag file in {}", fateId, info.bulkDir); + log.debug("{} removing the bulkDir processing flag file in {}", fateId, bulkInfo.bulkDir); Ample ample = env.getContext().getAmple(); - Path bulkDir = new Path(info.bulkDir); + Path bulkDir = new Path(bulkInfo.bulkDir); ample.removeBulkLoadInProgressFlag( "/" + bulkDir.getParent().getName() + "/" + bulkDir.getName()); - ample.putGcFileAndDirCandidates(info.tableId, - Collections.singleton(ReferenceFile.forFile(info.tableId, bulkDir))); + ample.putGcFileAndDirCandidates(bulkInfo.tableId, + Collections.singleton(ReferenceFile.forFile(bulkInfo.tableId, bulkDir))); - Text firstSplit = info.firstSplit == null ? null : new Text(info.firstSplit); - Text lastSplit = info.lastSplit == null ? null : new Text(info.lastSplit); + Text firstSplit = bulkInfo.firstSplit == null ? null : new Text(bulkInfo.firstSplit); + Text lastSplit = bulkInfo.lastSplit == null ? null : new Text(bulkInfo.lastSplit); log.debug("{} removing the metadata table markers for loaded files in range {} {}", fateId, firstSplit, lastSplit); - removeBulkLoadEntries(ample, info.tableId, fateId, firstSplit, lastSplit); + removeBulkLoadEntries(ample, bulkInfo.tableId, fateId, firstSplit, lastSplit); - Utils.unreserveHdfsDirectory(env.getContext(), info.sourceDir, fateId); - Utils.getReadLock(env.getContext(), info.tableId, fateId, LockRange.infinite()).unlock(); + Utils.unreserveHdfsDirectory(env.getContext(), bulkInfo.sourceDir, fateId); + Utils.getReadLock(env.getContext(), bulkInfo.tableId, fateId, LockRange.infinite()).unlock(); // delete json renames and mapping files Path renamingFile = new Path(bulkDir, Constants.BULK_RENAME_FILE); Path mappingFile = new Path(bulkDir, Constants.BULK_LOAD_MAPPING); @@ -89,7 +85,6 @@ public Repo call(FateId fateId, FateEnv env) throws Exception { } log.debug("completing bulkDir import transaction " + fateId); - env.removeBulkImportStatus(info.sourceDir); return null; } @@ -145,4 +140,9 @@ private static void removeBulkLoadEntries(Ample ample, TableId tableId, FateId f } } } + + @Override + public BulkState getState() { + return BulkState.CLEANING; + } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/ComputeBulkRange.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/ComputeBulkRange.java index 3eab86672af..afdc1697644 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/ComputeBulkRange.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/ComputeBulkRange.java @@ -26,27 +26,21 @@ import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.fate.FateId; import org.apache.accumulo.core.fate.Repo; -import org.apache.accumulo.manager.tableOps.AbstractFateOperation; import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.util.bulkCommand.ListBulk.BulkState; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class ComputeBulkRange extends AbstractFateOperation { +public class ComputeBulkRange extends AbstractBulkFateOperation { private static final long serialVersionUID = 1L; private static final Logger log = LoggerFactory.getLogger(ComputeBulkRange.class); - private final BulkInfo bulkInfo; - public ComputeBulkRange(TableId tableId, String sourceDir, boolean setTime) { - BulkInfo info = new BulkInfo(); - info.tableId = tableId; - info.sourceDir = sourceDir; - info.setTime = setTime; - this.bulkInfo = info; + super(BulkInfo.create(tableId, sourceDir, setTime)); } @Override @@ -72,4 +66,9 @@ public Repo call(FateId fateId, FateEnv env) throws Exception { return new PrepBulkImport(bulkInfo); } } + + @Override + public BulkState getState() { + return BulkState.PREPARING; + } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java index dae7fe4b2f2..75c71f02229 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java @@ -50,7 +50,6 @@ import org.apache.accumulo.core.fate.FateId; import org.apache.accumulo.core.fate.Repo; import org.apache.accumulo.core.logging.TabletLogger; -import org.apache.accumulo.core.manager.thrift.BulkImportState; import org.apache.accumulo.core.metadata.ReferencedTabletFile; import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.accumulo.core.metadata.TabletFile; @@ -66,10 +65,10 @@ import org.apache.accumulo.core.trace.TraceUtil; import org.apache.accumulo.core.util.PeekingIterator; import org.apache.accumulo.core.util.Timer; -import org.apache.accumulo.manager.tableOps.AbstractFateOperation; import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.server.fs.VolumeManager; import org.apache.accumulo.server.tablets.TabletTime; +import org.apache.accumulo.server.util.bulkCommand.ListBulk.BulkState; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.thrift.TException; @@ -83,7 +82,7 @@ * Make asynchronous load calls to each overlapping Tablet. This RepO does its work on the isReady * and will return a linear sleep value based on the largest number of Tablets on a TabletServer. */ -class LoadFiles extends AbstractFateOperation { +class LoadFiles extends AbstractBulkFateOperation { // visible for testing interface TabletsMetadataFactory { @@ -96,10 +95,8 @@ interface TabletsMetadataFactory { private static final Logger log = LoggerFactory.getLogger(LoadFiles.class); - private final BulkInfo bulkInfo; - public LoadFiles(BulkInfo bulkInfo) { - this.bulkInfo = bulkInfo; + super(bulkInfo); } @Override @@ -112,7 +109,6 @@ public long isReady(FateId fateId, FateEnv env) throws Exception { } VolumeManager fs = env.getVolumeManager(); final Path bulkDir = new Path(bulkInfo.bulkDir); - env.updateBulkImportStatus(bulkInfo.sourceDir, BulkImportState.LOADING); try (LoadMappingIterator lmi = BulkSerialize.getUpdatedLoadMapping(bulkDir.toString(), bulkInfo.tableId, fs::open)) { @@ -525,4 +521,9 @@ static List findOverlappingTablets(String fmtTid, KeyExtent load throw ne2; } } + + @Override + public BulkState getState() { + return BulkState.LOADING; + } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/PrepBulkImport.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/PrepBulkImport.java index 78a2303318e..c6e62c975e6 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/PrepBulkImport.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/PrepBulkImport.java @@ -51,12 +51,12 @@ import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletsMetadata; import org.apache.accumulo.core.util.PeekingIterator; -import org.apache.accumulo.manager.tableOps.AbstractFateOperation; import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.manager.tableOps.Utils; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.fs.VolumeManager; import org.apache.accumulo.server.tablets.UniqueNameAllocator; +import org.apache.accumulo.server.util.bulkCommand.ListBulk.BulkState; import org.apache.commons.io.FilenameUtils; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; @@ -75,16 +75,14 @@ * * @since 2.0.0 */ -public class PrepBulkImport extends AbstractFateOperation { +public class PrepBulkImport extends AbstractBulkFateOperation { private static final long serialVersionUID = 1L; private static final Logger log = LoggerFactory.getLogger(PrepBulkImport.class); - private final BulkInfo bulkInfo; - public PrepBulkImport(BulkInfo info) { - this.bulkInfo = info; + super(info); } @Override @@ -103,6 +101,11 @@ public long isReady(FateId fateId, FateEnv env) throws Exception { return Utils.reserveHdfsDirectory(env.getContext(), bulkInfo.sourceDir, fateId); } + @Override + public BulkState getState() { + return BulkState.PREPARING; + } + @VisibleForTesting interface TabletIterFactory extends AutoCloseable { Iterator newTabletIter(Text startRow); @@ -325,6 +328,5 @@ public void undo(FateId fateId, FateEnv environment) throws Exception { Utils.unreserveHdfsDirectory(environment.getContext(), bulkInfo.sourceDir, fateId); Utils.getReadLock(environment.getContext(), bulkInfo.tableId, fateId, LockRange.infinite()) .unlock(); - environment.removeBulkImportStatus(bulkInfo.sourceDir); } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/RefreshTablets.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/RefreshTablets.java index 477eac19763..0c3d042a249 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/RefreshTablets.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/RefreshTablets.java @@ -20,8 +20,8 @@ import org.apache.accumulo.core.fate.FateId; import org.apache.accumulo.core.fate.Repo; -import org.apache.accumulo.manager.tableOps.AbstractFateOperation; import org.apache.accumulo.manager.tableOps.FateEnv; +import org.apache.accumulo.server.util.bulkCommand.ListBulk.BulkState; /** * This Repo asks hosted tablets that were bulk loaded into to refresh their metadata. It works by @@ -30,14 +30,12 @@ * location its ok. That means the tablet either unloaded before of after the snapshot. In either * case the tablet will see the bulk files the next time its hosted somewhere. */ -public class RefreshTablets extends AbstractFateOperation { +public class RefreshTablets extends AbstractBulkFateOperation { private static final long serialVersionUID = 1L; - private final BulkInfo bulkInfo; - public RefreshTablets(BulkInfo bulkInfo) { - this.bulkInfo = bulkInfo; + super(bulkInfo); } @Override @@ -53,4 +51,9 @@ public Repo call(FateId fateId, FateEnv env) throws Exception { return new CleanUpBulkImport(bulkInfo); } + + @Override + public BulkState getState() { + return BulkState.REFRESHING; + } } diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportInformation.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportInformation.java index 39ed451884f..09197510707 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportInformation.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportInformation.java @@ -18,7 +18,7 @@ */ package org.apache.accumulo.monitor.rest.bulkImports; -import org.apache.accumulo.core.manager.thrift.BulkImportState; +import org.apache.accumulo.server.util.bulkCommand.ListBulk; /** * Stores bulk import in a JSON object @@ -30,7 +30,7 @@ public class BulkImportInformation { // Variable names become JSON key public String filename; public long age; - public BulkImportState state; + public ListBulk.BulkState state; public BulkImportInformation() {} @@ -41,7 +41,7 @@ public BulkImportInformation() {} * @param age age of the bulk import * @param state state of the bulk import */ - public BulkImportInformation(String filename, long age, BulkImportState state) { + public BulkImportInformation(String filename, long age, ListBulk.BulkState state) { this.filename = filename; this.age = age; this.state = state; diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportResource.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportResource.java index 2a5cdf9cc7f..7eea08ed315 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportResource.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportResource.java @@ -24,9 +24,8 @@ import jakarta.ws.rs.Produces; import jakarta.ws.rs.core.MediaType; -import org.apache.accumulo.core.manager.thrift.BulkImportStatus; -import org.apache.accumulo.core.manager.thrift.ManagerMonitorInfo; import org.apache.accumulo.monitor.Monitor; +import org.apache.accumulo.server.util.bulkCommand.ListBulk; /** * The BulkImportResource is responsible for obtaining the information of the bulk import, and @@ -49,16 +48,10 @@ public class BulkImportResource { @GET public BulkImport getTables() { BulkImport bulkImport = new BulkImport(); - ManagerMonitorInfo mmi = monitor.getMmi(); - if (mmi == null) { - return bulkImport; - } - - // Generating Bulk Import and adding it to the return object - for (BulkImportStatus bulk : mmi.bulkImports) { - bulkImport - .addBulkImport(new BulkImportInformation(bulk.filename, bulk.startTime, bulk.state)); - } + ListBulk.list(monitor.getContext(), bulkStatus -> { + bulkImport.addBulkImport(new BulkImportInformation(bulkStatus.sourceDir(), + bulkStatus.lastUpdate().toEpochMilli(), bulkStatus.state())); + }); return bulkImport; } diff --git a/shell/pom.xml b/shell/pom.xml index 656d9bfb396..23afddc6bdf 100644 --- a/shell/pom.xml +++ b/shell/pom.xml @@ -82,10 +82,6 @@ org.apache.hadoop hadoop-client-api - - org.apache.thrift - libthrift - org.jline jline diff --git a/shell/src/main/java/org/apache/accumulo/shell/Shell.java b/shell/src/main/java/org/apache/accumulo/shell/Shell.java index f1b939889be..a1ad2f73dbb 100644 --- a/shell/src/main/java/org/apache/accumulo/shell/Shell.java +++ b/shell/src/main/java/org/apache/accumulo/shell/Shell.java @@ -131,7 +131,6 @@ import org.apache.accumulo.shell.commands.ImportTableCommand; import org.apache.accumulo.shell.commands.InfoCommand; import org.apache.accumulo.shell.commands.InsertCommand; -import org.apache.accumulo.shell.commands.ListBulkCommand; import org.apache.accumulo.shell.commands.ListCompactionsCommand; import org.apache.accumulo.shell.commands.ListIterCommand; import org.apache.accumulo.shell.commands.ListResourceGroupsCommand; @@ -384,7 +383,7 @@ public boolean config(JCommander jc, ShellOptionsJC options) throws IOException new InsertCommand(), new MaxRowCommand(), new ScanCommand()}; Command[] debuggingCommands = {new ClasspathCommand(), new ListScansCommand(), new ListCompactionsCommand(), - new TraceCommand(), new PingCommand(), new ListBulkCommand(), new ListTabletsCommand()}; + new TraceCommand(), new PingCommand(), new ListTabletsCommand()}; Command[] execCommands = {new ExecfileCommand(), new HistoryCommand(), new ExtensionCommand()}; Command[] exitCommands = {new ByeCommand(), new ExitCommand(), new QuitCommand()}; Command[] helpCommands = diff --git a/shell/src/main/java/org/apache/accumulo/shell/commands/BulkImportListIterator.java b/shell/src/main/java/org/apache/accumulo/shell/commands/BulkImportListIterator.java deleted file mode 100644 index 22432925669..00000000000 --- a/shell/src/main/java/org/apache/accumulo/shell/commands/BulkImportListIterator.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.shell.commands; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.accumulo.core.manager.thrift.BulkImportStatus; -import org.apache.accumulo.core.manager.thrift.ManagerMonitorInfo; -import org.apache.accumulo.core.util.DurationFormat; - -public class BulkImportListIterator implements Iterator { - - private final Iterator iter; - - public BulkImportListIterator(ManagerMonitorInfo stats) { - List result = new ArrayList<>(); - for (BulkImportStatus status : stats.getBulkImports()) { - result.add(format(status)); - } - iter = result.iterator(); - } - - private String format(BulkImportStatus status) { - long diff = System.currentTimeMillis() - status.startTime; - var dur = new DurationFormat(diff, " "); - return String.format("%25s | %4s | %s", status.filename, dur, status.state); - } - - @Override - public boolean hasNext() { - return iter.hasNext(); - } - - @Override - public String next() { - return iter.next(); - } - - @Override - public void remove() { - iter.remove(); - } - -} diff --git a/shell/src/main/java/org/apache/accumulo/shell/commands/ListBulkCommand.java b/shell/src/main/java/org/apache/accumulo/shell/commands/ListBulkCommand.java deleted file mode 100644 index 117747c4958..00000000000 --- a/shell/src/main/java/org/apache/accumulo/shell/commands/ListBulkCommand.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.shell.commands; - -import org.apache.accumulo.core.clientImpl.ClientContext; -import org.apache.accumulo.core.manager.thrift.ManagerMonitorInfo; -import org.apache.accumulo.core.rpc.clients.ThriftClientTypes; -import org.apache.accumulo.core.trace.TraceUtil; -import org.apache.accumulo.shell.Shell; -import org.apache.accumulo.shell.Shell.Command; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; - -public class ListBulkCommand extends Command { - - private Option tserverOption; - private Option disablePaginationOpt; - - @Override - public String description() { - return "lists what bulk imports are currently running in accumulo."; - } - - @Override - public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) - throws Exception { - - ClientContext context = shellState.getContext(); - ManagerMonitorInfo stats = ThriftClientTypes.MANAGER.execute(context, - client -> client.getManagerStats(TraceUtil.traceInfo(), context.rpcCreds())); - - final boolean paginate = !cl.hasOption(disablePaginationOpt.getOpt()); - - if (cl.hasOption(tserverOption.getOpt())) { - shellState.getWriter().print(tserverOption.getOpt() - + " option is deprecated and will be removed in a future release.\n"); - shellState.getWriter().flush(); - } - - shellState.printLines(new BulkImportListIterator(stats), paginate); - return 0; - } - - @Override - public int numArgs() { - return 0; - } - - @Override - public Options getOptions() { - final Options opts = new Options(); - - tserverOption = new Option("ts", "tabletServer", true, "tablet server to list bulk imports"); - tserverOption.setArgName("tablet server"); - opts.addOption(tserverOption); - - disablePaginationOpt = new Option("np", "no-pagination", false, "disable pagination of output"); - opts.addOption(disablePaginationOpt); - - return opts; - } - -} diff --git a/shell/src/test/java/org/apache/accumulo/shell/commands/ListBulkCommandTest.java b/shell/src/test/java/org/apache/accumulo/shell/commands/ListBulkCommandTest.java deleted file mode 100644 index 86dee19ff6c..00000000000 --- a/shell/src/test/java/org/apache/accumulo/shell/commands/ListBulkCommandTest.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.shell.commands; - -import static org.easymock.EasyMock.createMock; -import static org.easymock.EasyMock.expect; -import static org.easymock.EasyMock.replay; -import static org.easymock.EasyMock.verify; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.accumulo.core.manager.thrift.BulkImportState; -import org.apache.accumulo.core.manager.thrift.BulkImportStatus; -import org.apache.accumulo.core.manager.thrift.ManagerMonitorInfo; -import org.junit.jupiter.api.Test; - -public class ListBulkCommandTest { - - /** - * Test the iterator used by BulkCommand prints correctly - */ - @Test - public void testBulkImportListIterator() { - ManagerMonitorInfo mmi = createMock(ManagerMonitorInfo.class); - List bulkImports = new ArrayList<>(); - - // tserver 1 - var bis1 = new BulkImportStatus(); - bis1.filename = "file1"; - bis1.startTime = System.currentTimeMillis(); - bis1.state = BulkImportState.COPY_FILES; - - // tserver 2 - var bis2 = new BulkImportStatus(); - bis2.filename = "file2"; - bis2.startTime = System.currentTimeMillis(); - bis2.state = BulkImportState.LOADING; - - bulkImports.add(bis1); - bulkImports.add(bis2); - - expect(mmi.getBulkImports()).andReturn(bulkImports).once(); - - replay(mmi); - - var iter = new BulkImportListIterator(mmi); - List printed = printLines(iter); - - assertTrue(printed.get(0).stripLeading().startsWith("file1")); - assertTrue(printed.get(0).endsWith("COPY_FILES")); - assertTrue(printed.get(1).stripLeading().startsWith("file2")); - assertTrue(printed.get(1).endsWith("LOADING")); - - verify(mmi); - } - - private List printLines(Iterator lines) { - List printed = new ArrayList<>(); - - while (lines.hasNext()) { - String nextLine = lines.next(); - if (nextLine == null) { - continue; - } - printed.add(nextLine); - } - return printed; - } -} diff --git a/test/src/main/java/org/apache/accumulo/test/GetManagerStats.java b/test/src/main/java/org/apache/accumulo/test/GetManagerStats.java index 21ff2e03a93..17482ab2914 100644 --- a/test/src/main/java/org/apache/accumulo/test/GetManagerStats.java +++ b/test/src/main/java/org/apache/accumulo/test/GetManagerStats.java @@ -23,7 +23,6 @@ import java.util.Map.Entry; import org.apache.accumulo.core.conf.SiteConfiguration; -import org.apache.accumulo.core.manager.thrift.BulkImportStatus; import org.apache.accumulo.core.manager.thrift.DeadServer; import org.apache.accumulo.core.manager.thrift.ManagerMonitorInfo; import org.apache.accumulo.core.manager.thrift.RecoveryStatus; @@ -62,12 +61,6 @@ public static void main(String[] args) throws Exception { out(2, "Last report: %s", new SimpleDateFormat().format(new Date(dead.lastStatus))); out(2, "Cause: %s", dead.status); } - out(0, "Bulk imports: %s", stats.bulkImports.size()); - for (BulkImportStatus bulk : stats.bulkImports) { - out(1, "Import directory: %s", bulk.filename); - out(2, "Bulk state %s", bulk.state); - out(2, "Bulk start %s", bulk.startTime); - } if (stats.tableMap != null && !stats.tableMap.isEmpty()) { out(0, "Tables"); for (Entry entry : stats.tableMap.entrySet()) { @@ -117,13 +110,6 @@ public static void main(String[] args) throws Exception { out(3, "Progress: %.2f%%", sort.progress * 100); out(3, "Time running: %s", sort.runtime / 1000.); } - out(3, "Bulk imports: %s", stats.bulkImports.size()); - for (BulkImportStatus bulk : stats.bulkImports) { - out(4, "Import file: %s", bulk.filename); - out(5, "Bulk state %s", bulk.state); - out(5, "Bulk start %s", bulk.startTime); - } - } } } diff --git a/test/src/main/java/org/apache/accumulo/test/fate/FlakyFateManager.java b/test/src/main/java/org/apache/accumulo/test/fate/FlakyFateManager.java index d8ea9578346..be9a21c7356 100644 --- a/test/src/main/java/org/apache/accumulo/test/fate/FlakyFateManager.java +++ b/test/src/main/java/org/apache/accumulo/test/fate/FlakyFateManager.java @@ -23,9 +23,9 @@ import org.apache.accumulo.core.cli.ServerOpts; import org.apache.accumulo.core.fate.Fate; import org.apache.accumulo.core.fate.FateStore; +import org.apache.accumulo.core.fate.TraceRepo; import org.apache.accumulo.manager.Manager; import org.apache.accumulo.manager.tableOps.FateEnv; -import org.apache.accumulo.manager.tableOps.TraceRepo; import org.apache.accumulo.server.ServerContext; import org.slf4j.LoggerFactory; diff --git a/test/src/main/java/org/apache/accumulo/test/fate/SlowFateSplitManager.java b/test/src/main/java/org/apache/accumulo/test/fate/SlowFateSplitManager.java index 1b68fec1c8e..b33a47044a9 100644 --- a/test/src/main/java/org/apache/accumulo/test/fate/SlowFateSplitManager.java +++ b/test/src/main/java/org/apache/accumulo/test/fate/SlowFateSplitManager.java @@ -23,9 +23,9 @@ import org.apache.accumulo.core.cli.ServerOpts; import org.apache.accumulo.core.fate.Fate; import org.apache.accumulo.core.fate.FateStore; +import org.apache.accumulo.core.fate.TraceRepo; import org.apache.accumulo.manager.Manager; import org.apache.accumulo.manager.tableOps.FateEnv; -import org.apache.accumulo.manager.tableOps.TraceRepo; import org.apache.accumulo.server.ServerContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory;