001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs.server.namenode.snapshot;
019
020 import java.io.DataInput;
021 import java.io.DataOutput;
022 import java.io.IOException;
023 import java.util.ArrayList;
024 import java.util.Collections;
025 import java.util.HashMap;
026 import java.util.List;
027 import java.util.Map;
028 import java.util.concurrent.atomic.AtomicInteger;
029
030 import javax.management.ObjectName;
031
032 import org.apache.hadoop.hdfs.DFSUtil;
033 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
034 import org.apache.hadoop.hdfs.protocol.SnapshotException;
035 import org.apache.hadoop.hdfs.protocol.SnapshotInfo;
036 import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
037 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
038 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
039 import org.apache.hadoop.hdfs.server.namenode.FSImageFormat;
040 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
041 import org.apache.hadoop.hdfs.server.namenode.INode;
042 import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
043 import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
044 import org.apache.hadoop.hdfs.server.namenode.INodesInPath;
045 import org.apache.hadoop.metrics2.util.MBeans;
046
047 import com.google.common.base.Preconditions;
048
049 /**
050 * Manage snapshottable directories and their snapshots.
051 *
052 * This class includes operations that create, access, modify snapshots and/or
053 * snapshot-related data. In general, the locking structure of snapshot
054 * operations is: <br>
055 *
056 * 1. Lock the {@link FSNamesystem} lock in {@link FSNamesystem} before calling
057 * into {@link SnapshotManager} methods.<br>
058 * 2. Lock the {@link FSDirectory} lock for the {@link SnapshotManager} methods
059 * if necessary.
060 */
061 public class SnapshotManager implements SnapshotStatsMXBean {
062 private boolean allowNestedSnapshots = false;
063 private final FSDirectory fsdir;
064 private static final int SNAPSHOT_ID_BIT_WIDTH = 24;
065
066 private final AtomicInteger numSnapshots = new AtomicInteger();
067
068 private int snapshotCounter = 0;
069
070 /** All snapshottable directories in the namesystem. */
071 private final Map<Long, INodeDirectory> snapshottables =
072 new HashMap<Long, INodeDirectory>();
073
074 public SnapshotManager(final FSDirectory fsdir) {
075 this.fsdir = fsdir;
076 }
077
078 /** Used in tests only */
079 void setAllowNestedSnapshots(boolean allowNestedSnapshots) {
080 this.allowNestedSnapshots = allowNestedSnapshots;
081 }
082
083 private void checkNestedSnapshottable(INodeDirectory dir, String path)
084 throws SnapshotException {
085 if (allowNestedSnapshots) {
086 return;
087 }
088
089 for(INodeDirectory s : snapshottables.values()) {
090 if (s.isAncestorDirectory(dir)) {
091 throw new SnapshotException(
092 "Nested snapshottable directories not allowed: path=" + path
093 + ", the subdirectory " + s.getFullPathName()
094 + " is already a snapshottable directory.");
095 }
096 if (dir.isAncestorDirectory(s)) {
097 throw new SnapshotException(
098 "Nested snapshottable directories not allowed: path=" + path
099 + ", the ancestor " + s.getFullPathName()
100 + " is already a snapshottable directory.");
101 }
102 }
103 }
104
105 /**
106 * Set the given directory as a snapshottable directory.
107 * If the path is already a snapshottable directory, update the quota.
108 */
109 public void setSnapshottable(final String path, boolean checkNestedSnapshottable)
110 throws IOException {
111 final INodesInPath iip = fsdir.getINodesInPath4Write(path);
112 final INodeDirectory d = INodeDirectory.valueOf(iip.getLastINode(), path);
113 if (checkNestedSnapshottable) {
114 checkNestedSnapshottable(d, path);
115 }
116
117 if (d.isSnapshottable()) {
118 //The directory is already a snapshottable directory.
119 d.setSnapshotQuota(DirectorySnapshottableFeature.SNAPSHOT_LIMIT);
120 } else {
121 d.addSnapshottableFeature();
122 }
123 addSnapshottable(d);
124 }
125
126 /** Add the given snapshottable directory to {@link #snapshottables}. */
127 public void addSnapshottable(INodeDirectory dir) {
128 Preconditions.checkArgument(dir.isSnapshottable());
129 snapshottables.put(dir.getId(), dir);
130 }
131
132 /** Remove the given snapshottable directory from {@link #snapshottables}. */
133 private void removeSnapshottable(INodeDirectory s) {
134 snapshottables.remove(s.getId());
135 }
136
137 /** Remove snapshottable directories from {@link #snapshottables} */
138 public void removeSnapshottable(List<INodeDirectory> toRemove) {
139 if (toRemove != null) {
140 for (INodeDirectory s : toRemove) {
141 removeSnapshottable(s);
142 }
143 }
144 }
145
146 /**
147 * Set the given snapshottable directory to non-snapshottable.
148 *
149 * @throws SnapshotException if there are snapshots in the directory.
150 */
151 public void resetSnapshottable(final String path) throws IOException {
152 final INodesInPath iip = fsdir.getINodesInPath4Write(path);
153 final INodeDirectory d = INodeDirectory.valueOf(iip.getLastINode(), path);
154 DirectorySnapshottableFeature sf = d.getDirectorySnapshottableFeature();
155 if (sf == null) {
156 // the directory is already non-snapshottable
157 return;
158 }
159 if (sf.getNumSnapshots() > 0) {
160 throw new SnapshotException("The directory " + path + " has snapshot(s). "
161 + "Please redo the operation after removing all the snapshots.");
162 }
163
164 if (d == fsdir.getRoot()) {
165 d.setSnapshotQuota(0);
166 } else {
167 d.removeSnapshottableFeature();
168 }
169 removeSnapshottable(d);
170 }
171
172 /**
173 * Find the source root directory where the snapshot will be taken
174 * for a given path.
175 *
176 * @param path The directory path where the snapshot will be taken.
177 * @return Snapshottable directory.
178 * @throws IOException
179 * Throw IOException when the given path does not lead to an
180 * existing snapshottable directory.
181 */
182 public INodeDirectory getSnapshottableRoot(final String path)
183 throws IOException {
184 final INodeDirectory dir = INodeDirectory.valueOf(fsdir
185 .getINodesInPath4Write(path).getLastINode(), path);
186 if (!dir.isSnapshottable()) {
187 throw new SnapshotException(
188 "Directory is not a snapshottable directory: " + path);
189 }
190 return dir;
191 }
192
193 /**
194 * Create a snapshot of the given path.
195 * It is assumed that the caller will perform synchronization.
196 *
197 * @param path
198 * The directory path where the snapshot will be taken.
199 * @param snapshotName
200 * The name of the snapshot.
201 * @throws IOException
202 * Throw IOException when 1) the given path does not lead to an
203 * existing snapshottable directory, and/or 2) there exists a
204 * snapshot with the given name for the directory, and/or 3)
205 * snapshot number exceeds quota
206 */
207 public String createSnapshot(final String path, String snapshotName
208 ) throws IOException {
209 INodeDirectory srcRoot = getSnapshottableRoot(path);
210
211 if (snapshotCounter == getMaxSnapshotID()) {
212 // We have reached the maximum allowable snapshot ID and since we don't
213 // handle rollover we will fail all subsequent snapshot creation
214 // requests.
215 //
216 throw new SnapshotException(
217 "Failed to create the snapshot. The FileSystem has run out of " +
218 "snapshot IDs and ID rollover is not supported.");
219 }
220
221 srcRoot.addSnapshot(snapshotCounter, snapshotName);
222
223 //create success, update id
224 snapshotCounter++;
225 numSnapshots.getAndIncrement();
226 return Snapshot.getSnapshotPath(path, snapshotName);
227 }
228
229 /**
230 * Delete a snapshot for a snapshottable directory
231 * @param path Path to the directory where the snapshot was taken
232 * @param snapshotName Name of the snapshot to be deleted
233 * @param collectedBlocks Used to collect information to update blocksMap
234 * @throws IOException
235 */
236 public void deleteSnapshot(final String path, final String snapshotName,
237 BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes)
238 throws IOException {
239 // parse the path, and check if the path is a snapshot path
240 // the INodeDirectorySnapshottable#valueOf method will throw Exception
241 // if the path is not for a snapshottable directory
242 INodeDirectory srcRoot = getSnapshottableRoot(path);
243 srcRoot.removeSnapshot(snapshotName, collectedBlocks, removedINodes);
244 numSnapshots.getAndDecrement();
245 }
246
247 /**
248 * Rename the given snapshot
249 * @param path
250 * The directory path where the snapshot was taken
251 * @param oldSnapshotName
252 * Old name of the snapshot
253 * @param newSnapshotName
254 * New name of the snapshot
255 * @throws IOException
256 * Throw IOException when 1) the given path does not lead to an
257 * existing snapshottable directory, and/or 2) the snapshot with the
258 * old name does not exist for the directory, and/or 3) there exists
259 * a snapshot with the new name for the directory
260 */
261 public void renameSnapshot(final String path, final String oldSnapshotName,
262 final String newSnapshotName) throws IOException {
263 // Find the source root directory path where the snapshot was taken.
264 // All the check for path has been included in the valueOf method.
265 final INodeDirectory srcRoot = getSnapshottableRoot(path);
266 // Note that renameSnapshot and createSnapshot are synchronized externally
267 // through FSNamesystem's write lock
268 srcRoot.renameSnapshot(path, oldSnapshotName, newSnapshotName);
269 }
270
271 public int getNumSnapshottableDirs() {
272 return snapshottables.size();
273 }
274
275 public int getNumSnapshots() {
276 return numSnapshots.get();
277 }
278
279 void setNumSnapshots(int num) {
280 numSnapshots.set(num);
281 }
282
283 int getSnapshotCounter() {
284 return snapshotCounter;
285 }
286
287 void setSnapshotCounter(int counter) {
288 snapshotCounter = counter;
289 }
290
291 INodeDirectory[] getSnapshottableDirs() {
292 return snapshottables.values().toArray(
293 new INodeDirectory[snapshottables.size()]);
294 }
295
296 /**
297 * Write {@link #snapshotCounter}, {@link #numSnapshots},
298 * and all snapshots to the DataOutput.
299 */
300 public void write(DataOutput out) throws IOException {
301 out.writeInt(snapshotCounter);
302 out.writeInt(numSnapshots.get());
303
304 // write all snapshots.
305 for(INodeDirectory snapshottableDir : snapshottables.values()) {
306 for (Snapshot s : snapshottableDir.getDirectorySnapshottableFeature()
307 .getSnapshotList()) {
308 s.write(out);
309 }
310 }
311 }
312
313 /**
314 * Read values of {@link #snapshotCounter}, {@link #numSnapshots}, and
315 * all snapshots from the DataInput
316 */
317 public Map<Integer, Snapshot> read(DataInput in, FSImageFormat.Loader loader
318 ) throws IOException {
319 snapshotCounter = in.readInt();
320 numSnapshots.set(in.readInt());
321
322 // read snapshots
323 final Map<Integer, Snapshot> snapshotMap = new HashMap<Integer, Snapshot>();
324 for(int i = 0; i < numSnapshots.get(); i++) {
325 final Snapshot s = Snapshot.read(in, loader);
326 snapshotMap.put(s.getId(), s);
327 }
328 return snapshotMap;
329 }
330
331 /**
332 * List all the snapshottable directories that are owned by the current user.
333 * @param userName Current user name.
334 * @return Snapshottable directories that are owned by the current user,
335 * represented as an array of {@link SnapshottableDirectoryStatus}. If
336 * {@code userName} is null, return all the snapshottable dirs.
337 */
338 public SnapshottableDirectoryStatus[] getSnapshottableDirListing(
339 String userName) {
340 if (snapshottables.isEmpty()) {
341 return null;
342 }
343
344 List<SnapshottableDirectoryStatus> statusList =
345 new ArrayList<SnapshottableDirectoryStatus>();
346 for (INodeDirectory dir : snapshottables.values()) {
347 if (userName == null || userName.equals(dir.getUserName())) {
348 SnapshottableDirectoryStatus status = new SnapshottableDirectoryStatus(
349 dir.getModificationTime(), dir.getAccessTime(),
350 dir.getFsPermission(), dir.getUserName(), dir.getGroupName(),
351 dir.getLocalNameBytes(), dir.getId(),
352 dir.getChildrenNum(Snapshot.CURRENT_STATE_ID),
353 dir.getDirectorySnapshottableFeature().getNumSnapshots(),
354 dir.getDirectorySnapshottableFeature().getSnapshotQuota(),
355 dir.getParent() == null ? DFSUtil.EMPTY_BYTES :
356 DFSUtil.string2Bytes(dir.getParent().getFullPathName()));
357 statusList.add(status);
358 }
359 }
360 Collections.sort(statusList, SnapshottableDirectoryStatus.COMPARATOR);
361 return statusList.toArray(
362 new SnapshottableDirectoryStatus[statusList.size()]);
363 }
364
365 /**
366 * Compute the difference between two snapshots of a directory, or between a
367 * snapshot of the directory and its current tree.
368 */
369 public SnapshotDiffReport diff(final String path, final String from,
370 final String to) throws IOException {
371 // Find the source root directory path where the snapshots were taken.
372 // All the check for path has been included in the valueOf method.
373 final INodeDirectory snapshotRoot = getSnapshottableRoot(path);
374
375 if ((from == null || from.isEmpty())
376 && (to == null || to.isEmpty())) {
377 // both fromSnapshot and toSnapshot indicate the current tree
378 return new SnapshotDiffReport(path, from, to,
379 Collections.<DiffReportEntry> emptyList());
380 }
381 final SnapshotDiffInfo diffs = snapshotRoot
382 .getDirectorySnapshottableFeature().computeDiff(snapshotRoot, from, to);
383 return diffs != null ? diffs.generateReport() : new SnapshotDiffReport(
384 path, from, to, Collections.<DiffReportEntry> emptyList());
385 }
386
387 public void clearSnapshottableDirs() {
388 snapshottables.clear();
389 }
390
391 /**
392 * Returns the maximum allowable snapshot ID based on the bit width of the
393 * snapshot ID.
394 *
395 * @return maximum allowable snapshot ID.
396 */
397 public int getMaxSnapshotID() {
398 return ((1 << SNAPSHOT_ID_BIT_WIDTH) - 1);
399 }
400
401 private ObjectName mxBeanName;
402
403 public void registerMXBean() {
404 mxBeanName = MBeans.register("NameNode", "SnapshotInfo", this);
405 }
406
407 public void shutdown() {
408 MBeans.unregister(mxBeanName);
409 mxBeanName = null;
410 }
411
412 @Override // SnapshotStatsMXBean
413 public SnapshottableDirectoryStatus.Bean[]
414 getSnapshottableDirectories() {
415 List<SnapshottableDirectoryStatus.Bean> beans =
416 new ArrayList<SnapshottableDirectoryStatus.Bean>();
417 for (INodeDirectory d : getSnapshottableDirs()) {
418 beans.add(toBean(d));
419 }
420 return beans.toArray(new SnapshottableDirectoryStatus.Bean[beans.size()]);
421 }
422
423 @Override // SnapshotStatsMXBean
424 public SnapshotInfo.Bean[] getSnapshots() {
425 List<SnapshotInfo.Bean> beans = new ArrayList<SnapshotInfo.Bean>();
426 for (INodeDirectory d : getSnapshottableDirs()) {
427 for (Snapshot s : d.getDirectorySnapshottableFeature().getSnapshotList()) {
428 beans.add(toBean(s));
429 }
430 }
431 return beans.toArray(new SnapshotInfo.Bean[beans.size()]);
432 }
433
434 public static SnapshottableDirectoryStatus.Bean toBean(INodeDirectory d) {
435 return new SnapshottableDirectoryStatus.Bean(
436 d.getFullPathName(),
437 d.getDirectorySnapshottableFeature().getNumSnapshots(),
438 d.getDirectorySnapshottableFeature().getSnapshotQuota(),
439 d.getModificationTime(),
440 Short.valueOf(Integer.toOctalString(
441 d.getFsPermissionShort())),
442 d.getUserName(),
443 d.getGroupName());
444 }
445
446 public static SnapshotInfo.Bean toBean(Snapshot s) {
447 return new SnapshotInfo.Bean(
448 s.getRoot().getLocalName(), s.getRoot().getFullPathName(),
449 s.getRoot().getModificationTime());
450 }
451 }