Cyberduck Mountain Duck CLI

Changeset 42426


Ignore:
Timestamp:
Aug 18, 2017 11:07:28 AM (10 months ago)
Author:
dkocher
Message:

Merge pull request #1025 in ITERATE/cyberduck from bugfix/TRAC-10055 to master

  • commit '0a6bfeff88ad7ff1279e5dc583dfe16a6d33fb9e': (26 commits) Fix tests. Fix test. Always use versioned list and fall back on interoperability failure. Review. Add test. Review. Implement for buckets. Review. Reuse attributes feature. Review. Review. Include versionId in return attributes. Always use versioned implementation. Extract super class. Change search predicate in list depending on version id in file attributes. Add tests. Add case insensitive search as filter. Review. Review. Remove unused. Always use versioned implementation. ...
Files:
5 added
1 deleted
21 edited

Legend:

Unmodified
Added
Removed
  • trunk

  • trunk/core/src/main/java/ch/cyberduck/core/DefaultPathPredicate.java

    r41511 r42426  
    9393
    9494    @Override
    95     public boolean test(final Path file) {
    96         return this.hashCode() == new DefaultPathPredicate(file).hashCode();
     95    public boolean test(final Path test) {
     96        return this.hashCode() == new DefaultPathPredicate(test).hashCode();
    9797    }
    9898}
  • trunk/core/src/main/java/ch/cyberduck/core/SimplePathPredicate.java

    r40738 r42426  
    3131
    3232    @Override
    33     public boolean test(final Path file) {
    34         return this.hashCode() == new SimplePathPredicate(file).hashCode();
     33    public boolean test(final Path test) {
     34        return this.hashCode() == new SimplePathPredicate(test).hashCode();
    3535    }
    3636}
  • trunk/core/src/main/java/ch/cyberduck/core/preferences/Preferences.java

    r42276 r42426  
    691691        defaults.put("s3.listing.chunksize", String.valueOf(1000));
    692692
    693         /*
    694           Show revisions as hidden files in browser
    695          */
    696         defaults.put("s3.revisions.enable", String.valueOf(true));
    697 
    698693        defaults.put("s3.upload.md5", String.valueOf(true));
    699694
  • trunk/core/src/main/java/ch/cyberduck/core/shared/DefaultAttributesFinderFeature.java

    r42095 r42426  
    2121import ch.cyberduck.core.AttributedList;
    2222import ch.cyberduck.core.Cache;
    23 import ch.cyberduck.core.DisabledListProgressListener;
    24 import ch.cyberduck.core.NullFilter;
    2523import ch.cyberduck.core.Path;
    2624import ch.cyberduck.core.PathAttributes;
    2725import ch.cyberduck.core.PathCache;
    2826import ch.cyberduck.core.Session;
    29 import ch.cyberduck.core.SimplePathPredicate;
    3027import ch.cyberduck.core.exception.AccessDeniedException;
    3128import ch.cyberduck.core.exception.BackgroundException;
     
    3633import org.apache.log4j.Logger;
    3734
    38 public class DefaultAttributesFinderFeature implements AttributesFinder {
     35public class DefaultAttributesFinderFeature extends ListFilteringFeature implements AttributesFinder {
    3936    private static final Logger log = Logger.getLogger(DefaultAttributesFinderFeature.class);
    4037
     
    4542
    4643    public DefaultAttributesFinderFeature(final Session<?> session) {
     44        super(session);
    4745        this.session = session;
    4846    }
     
    5351            return PathAttributes.EMPTY;
    5452        }
    55         final AttributedList<Path> list;
    56         if(!cache.isCached(file.getParent())) {
    57             try {
    58                 list = session.list(file.getParent(), new DisabledListProgressListener());
    59                 cache.put(file.getParent(), list);
     53        try {
     54            final Path found = this.search(file);
     55            final AttributedList<Path> list;
     56            if(null == found) {
     57                throw new NotfoundException(file.getAbsolute());
    6058            }
    61             catch(InteroperabilityException | AccessDeniedException | NotfoundException f) {
    62                 log.warn(String.format("Failure listing directory %s. %s", file.getParent(), f.getMessage()));
    63                 // Try native implementation
    64                 final AttributesFinder feature = session._getFeature(AttributesFinder.class);
    65                 if(feature instanceof DefaultAttributesFinderFeature) {
    66                     throw f;
    67                 }
    68                 return feature.withCache(cache).find(file);
     59            return found.attributes();
     60        }
     61        catch(InteroperabilityException | AccessDeniedException | NotfoundException f) {
     62            log.warn(String.format("Failure listing directory %s. %s", file.getParent(), f.getMessage()));
     63            // Try native implementation
     64            final AttributesFinder feature = session._getFeature(AttributesFinder.class);
     65            if(feature instanceof DefaultAttributesFinderFeature) {
     66                throw f;
    6967            }
     68            return feature.withCache(cache).find(file);
    7069        }
    71         else {
    72             list = cache.get(file.getParent());
    73         }
    74         final Path result = list.filter(new NullFilter<>()).find(new SimplePathPredicate(file));
    75         if(null == result) {
    76             throw new NotfoundException(file.getAbsolute());
    77         }
    78         return result.attributes();
    7970    }
    8071
     
    8273    public DefaultAttributesFinderFeature withCache(final Cache<Path> cache) {
    8374        this.cache = cache;
     75        super.withCache(cache);
    8476        return this;
    8577    }
  • trunk/core/src/main/java/ch/cyberduck/core/shared/DefaultFindFeature.java

    r42095 r42426  
    1919 */
    2020
    21 import ch.cyberduck.core.AttributedList;
    2221import ch.cyberduck.core.Cache;
    23 import ch.cyberduck.core.DisabledListProgressListener;
    24 import ch.cyberduck.core.NullFilter;
    2522import ch.cyberduck.core.Path;
    26 import ch.cyberduck.core.PathCache;
    2723import ch.cyberduck.core.Session;
    28 import ch.cyberduck.core.SimplePathPredicate;
    2924import ch.cyberduck.core.exception.BackgroundException;
    3025import ch.cyberduck.core.exception.NotfoundException;
    3126import ch.cyberduck.core.features.Find;
    3227
    33 import org.apache.commons.lang3.StringUtils;
    3428import org.apache.log4j.Logger;
    3529
    36 public class DefaultFindFeature implements Find {
     30public class DefaultFindFeature extends ListFilteringFeature implements Find {
    3731    private static final Logger log = Logger.getLogger(DefaultFindFeature.class);
    3832
    39     private final Session<?> session;
    40 
    41     private Cache<Path> cache
    42             = PathCache.empty();
    43 
    4433    public DefaultFindFeature(final Session<?> session) {
    45         this.session = session;
     34        super(session);
    4635    }
    4736
     
    5241        }
    5342        try {
    54             final AttributedList<Path> list;
    55             if(!cache.isCached(file.getParent())) {
    56                 list = session.list(file.getParent(), new DisabledListProgressListener());
    57                 cache.put(file.getParent(), list);
    58             }
    59             else {
    60                 list = cache.get(file.getParent());
    61             }
    62             final Path existing = list.filter(new NullFilter<>()).find(new DuplicateFilterPathPredicate(file));
    63             final boolean found = existing != null;
    64             if(!found) {
    65                 switch(session.getCase()) {
    66                     case insensitive:
    67                         // Find for all matching filenames ignoring case
    68                         for(Path f : list) {
    69                             if(!f.getType().equals(file.getType())) {
    70                                 continue;
    71                             }
    72                             if(StringUtils.equalsIgnoreCase(f.getName(), file.getName())) {
    73                                 log.warn(String.format("Found matching file %s ignoring case", f));
    74                                 return true;
    75                             }
    76                         }
    77                 }
    78             }
    79             return found;
     43            final Path found = this.search(file);
     44            return found != null;
    8045        }
    8146        catch(NotfoundException e) {
     
    8651    @Override
    8752    public DefaultFindFeature withCache(final Cache<Path> cache) {
    88         this.cache = cache;
     53        super.withCache(cache);
    8954        return this;
    9055    }
    91 
    92     private final class DuplicateFilterPathPredicate extends SimplePathPredicate {
    93 
    94         public DuplicateFilterPathPredicate(final Path file) {
    95             super(file);
    96         }
    97 
    98         @Override
    99         public boolean test(final Path file) {
    100             if(file.attributes().isDuplicate()) {
    101                 return false;
    102             }
    103             return super.test(file);
    104         }
    105     }
    10656}
  • trunk/s3/src/main/java/ch/cyberduck/core/s3/S3AccessControlListFeature.java

    r41497 r42426  
    2525import ch.cyberduck.core.exception.BackgroundException;
    2626import ch.cyberduck.core.features.AclPermission;
    27 import ch.cyberduck.core.features.Versioning;
    2827import ch.cyberduck.core.shared.DefaultAclFeature;
    2928
     
    6564            }
    6665            else if(file.isFile() || file.isPlaceholder()) {
    67                 final Versioning versioning = session.getFeature(Versioning.class);
    68                 if(versioning != null && versioning.getConfiguration(containerService.getContainer(file)).isEnabled()) {
    69                     final String version = file.attributes().getVersionId();
    70                     return this.convert(session.getClient().getVersionedObjectAcl(version,
    71                             containerService.getContainer(file).getName(), containerService.getKey(file)));
    72                 }
    73                 else {
    74                     // This method can be performed by anonymous services, but can only succeed if the
    75                     // object's existing ACL already allows read access by the anonymous user.
    76                     return this.convert(session.getClient().getObjectAcl(
    77                             containerService.getContainer(file).getName(), containerService.getKey(file)));
    78                 }
     66                return this.convert(session.getClient().getVersionedObjectAcl(file.attributes().getVersionId(),
     67                        containerService.getContainer(file).getName(), containerService.getKey(file)));
    7968            }
    8069            return Acl.EMPTY;
  • trunk/s3/src/main/java/ch/cyberduck/core/s3/S3AttributesFinderFeature.java

    r41497 r42426  
    2828import ch.cyberduck.core.features.AttributesFinder;
    2929import ch.cyberduck.core.features.Encryption;
    30 import ch.cyberduck.core.features.Versioning;
    3130import ch.cyberduck.core.io.Checksum;
    3231
     
    6059        }
    6160        if(containerService.isContainer(file)) {
    62             return PathAttributes.EMPTY;
     61            final PathAttributes attributes = new PathAttributes();
     62            attributes.setRegion(new S3LocationFeature(session).getLocation(file).getIdentifier());
     63            return attributes;
    6364        }
    64         else {
    65             return this.convert(this.details(file));
    66         }
     65        return this.convert(this.details(file));
    6766    }
    6867
     
    7574        final String container = containerService.getContainer(file).getName();
    7675        try {
    77             final Versioning versioning = session.getFeature(Versioning.class);
    78             if(versioning != null && versioning.getConfiguration(containerService.getContainer(file)).isEnabled()) {
    79                 final String version = file.attributes().getVersionId();
    80                 return session.getClient().getVersionedObjectDetails(version,
    81                         container, containerService.getKey(file));
    82             }
    83             else {
    84                 return session.getClient().getObjectDetails(container, containerService.getKey(file));
    85             }
     76            return session.getClient().getVersionedObjectDetails(file.attributes().getVersionId(),
     77                    container, containerService.getKey(file));
    8678        }
    8779        catch(ServiceException e) {
     
    9486                        // Fallback to GET if HEAD fails with 400 response
    9587                        try {
    96                             final S3Object object = session.getClient().getObject(containerService.getContainer(file).getName(),
    97                                     containerService.getKey(file), null, null, null, null, null, null);
     88                            final S3Object object = session.getClient().getVersionedObject(file.attributes().getVersionId(),
     89                                    containerService.getContainer(file).getName(), containerService.getKey(file));
    9890                            IOUtils.closeQuietly(object.getDataInputStream());
    9991                            return object;
  • trunk/s3/src/main/java/ch/cyberduck/core/s3/S3DefaultDeleteFeature.java

    r42292 r42426  
    7272                    try {
    7373                        // Always returning 204 even if the key does not exist. Does not return 404 for non-existing keys
    74                         session.getClient().deleteObject(containerService.getContainer(file).getName(), containerService.getKey(file));
     74                        session.getClient().deleteVersionedObject(file.attributes().getVersionId(),
     75                                containerService.getContainer(file).getName(), containerService.getKey(file));
    7576                    }
    7677                    catch(ServiceException e) {
  • trunk/s3/src/main/java/ch/cyberduck/core/s3/S3FindFeature.java

    r39836 r42426  
    2222import ch.cyberduck.core.Path;
    2323import ch.cyberduck.core.PathCache;
    24 import ch.cyberduck.core.PathContainerService;
     24import ch.cyberduck.core.exception.AccessDeniedException;
    2525import ch.cyberduck.core.exception.BackgroundException;
    26 import ch.cyberduck.core.exception.InteroperabilityException;
    2726import ch.cyberduck.core.exception.NotfoundException;
    2827import ch.cyberduck.core.features.Find;
    2928
    30 import org.apache.commons.io.IOUtils;
    31 import org.apache.http.HttpStatus;
    3229import org.apache.log4j.Logger;
    33 import org.jets3t.service.ServiceException;
    34 import org.jets3t.service.model.S3Object;
    3530
    3631public class S3FindFeature implements Find {
     
    3833
    3934    private final S3Session session;
    40 
    41     private final PathContainerService containerService
    42             = new S3PathContainerService();
    4335
    4436    private Cache<Path> cache;
     
    7163        }
    7264        try {
    73             if(session.getClient().isObjectInBucket(containerService.getContainer(file).getName(),
    74                     containerService.getKey(file))) {
    75                 list.add(file);
    76                 return true;
    77             }
    78             else {
    79                 list.attributes().addHidden(file);
    80                 return false;
    81             }
     65            new S3AttributesFinderFeature(session).withCache(cache).find(file);
     66            list.add(file);
     67            return true;
    8268        }
    83         catch(ServiceException e) {
    84             switch(session.getSignatureVersion()) {
    85                 case AWS4HMACSHA256:
    86                     if(new S3ExceptionMappingService().map(e) instanceof InteroperabilityException) {
    87                         log.warn("Workaround HEAD failure using GET because the expected AWS region cannot be determined " +
    88                                 "from the HEAD error message if using AWS4-HMAC-SHA256 with the wrong region specifier " +
    89                                 "in the authentication header.");
    90                         // Fallback to GET if HEAD fails with 400 response
    91                         try {
    92                             final S3Object object = session.getClient().getObject(containerService.getContainer(file).getName(),
    93                                     containerService.getKey(file), null, null, null, null, null, null);
    94                             IOUtils.closeQuietly(object.getDataInputStream());
    95                             list.add(file);
    96                             return true;
    97                         }
    98                         catch(ServiceException f) {
    99                             if(new S3ExceptionMappingService().map(f) instanceof NotfoundException) {
    100                                 list.attributes().addHidden(file);
    101                                 return false;
    102                             }
    103                             if(f.getResponseCode() == HttpStatus.SC_REQUESTED_RANGE_NOT_SATISFIABLE) {
    104                                 // A 0 byte content length file does exist but will return 416
    105                                 return true;
    106                             }
    107                         }
    108                     }
    109             }
    110             throw new S3ExceptionMappingService().map("Failure to read attributes of {0}", e, file);
     69        catch(NotfoundException e) {
     70            list.attributes().addHidden(file);
     71            return false;
     72        }
     73        catch(AccessDeniedException e) {
     74            list.add(file);
     75            // Object is inaccessible to current user, but does exist.
     76            return true;
    11177        }
    11278    }
  • trunk/s3/src/main/java/ch/cyberduck/core/s3/S3ObjectListService.java

    r41503 r42426  
    2424import ch.cyberduck.core.ListService;
    2525import ch.cyberduck.core.Path;
     26import ch.cyberduck.core.PathAttributes;
    2627import ch.cyberduck.core.PathContainerService;
    2728import ch.cyberduck.core.PathNormalizer;
     
    9091                            ? EnumSet.of(Path.Type.directory) : EnumSet.of(Path.Type.file);
    9192                    final Path file;
     93                    final PathAttributes attributes = this.attributes.convert(object);
     94                    // Copy bucket location
     95                    attributes.setRegion(bucket.attributes().getRegion());
    9296                    if(null == delimiter) {
    93                         file = new Path(String.format("%s%s%s", bucket.getAbsolute(), String.valueOf(Path.DELIMITER), key), types, attributes.convert(object));
     97                        file = new Path(String.format("%s%s%s", bucket.getAbsolute(), String.valueOf(Path.DELIMITER), key), types, attributes);
    9498                    }
    9599                    else {
    96                         file = new Path(directory, PathNormalizer.name(key), types, attributes.convert(object));
     100                        file = new Path(directory, PathNormalizer.name(key), types, attributes);
    97101                    }
    98                     // Copy bucket location
    99                     file.attributes().setRegion(bucket.attributes().getRegion());
    100102                    children.add(file);
    101103                }
     
    111113                    }
    112114                    final Path file;
     115                    final PathAttributes attributes = new PathAttributes();
    113116                    if(null == delimiter) {
    114                         file = new Path(String.format("%s%s%s", bucket.getAbsolute(), String.valueOf(Path.DELIMITER), key), EnumSet.of(Path.Type.directory, Path.Type.placeholder));
     117                        file = new Path(String.format("%s%s%s", bucket.getAbsolute(), String.valueOf(Path.DELIMITER), key), EnumSet.of(Path.Type.directory, Path.Type.placeholder), attributes);
    115118                    }
    116119                    else {
    117                         file = new Path(directory, PathNormalizer.name(key), EnumSet.of(Path.Type.directory, Path.Type.placeholder));
     120                        file = new Path(directory, PathNormalizer.name(key), EnumSet.of(Path.Type.directory, Path.Type.placeholder), attributes);
    118121                    }
    119                     file.attributes().setRegion(bucket.attributes().getRegion());
     122                    attributes.setRegion(bucket.attributes().getRegion());
    120123                    children.add(file);
    121124                }
  • trunk/s3/src/main/java/ch/cyberduck/core/s3/S3ReadFeature.java

    r42105 r42426  
    2424import ch.cyberduck.core.exception.BackgroundException;
    2525import ch.cyberduck.core.features.Read;
    26 import ch.cyberduck.core.features.Versioning;
    2726import ch.cyberduck.core.http.HttpRange;
    2827import ch.cyberduck.core.transfer.TransferStatus;
     
    4140
    4241    private final S3Session session;
    43     private final Versioning versioning;
    44 
    4542
    4643    public S3ReadFeature(final S3Session session) {
    47         this(session, session.getFeature(Versioning.class));
    48     }
    49 
    50     public S3ReadFeature(final S3Session session, final Versioning versioning) {
    5144        this.session = session;
    52         this.versioning = versioning;
    5345    }
    5446
     
    5648    public InputStream read(final Path file, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException {
    5749        try {
    58             final S3Object object;
    5950            final HttpRange range = HttpRange.withStatus(status);
    6051            final RequestEntityRestStorageService client = session.getClient();
    61             if(versioning != null && versioning.getConfiguration(containerService.getContainer(file)).isEnabled()) {
    62                 object = client.getVersionedObject(
    63                         file.attributes().getVersionId(),
    64                         containerService.getContainer(file).getName(), containerService.getKey(file),
    65                         null, // ifModifiedSince
    66                         null, // ifUnmodifiedSince
    67                         null, // ifMatch
    68                         null, // ifNoneMatch
    69                         status.isAppend() ? range.getStart() : null,
    70                         status.isAppend() ? (range.getEnd() == -1 ? null : range.getEnd()) : null);
    71             }
    72             else {
    73                 object = client.getObject(
    74                         containerService.getContainer(file).getName(),
    75                         containerService.getKey(file),
    76                         null, // ifModifiedSince
    77                         null, // ifUnmodifiedSince
    78                         null, // ifMatch
    79                         null, // ifNoneMatch
    80                         status.isAppend() ? range.getStart() : null,
    81                         status.isAppend() ? (range.getEnd() == -1 ? null : range.getEnd()) : null);
    82             }
     52            final S3Object object = client.getVersionedObject(
     53                    file.attributes().getVersionId(),
     54                    containerService.getContainer(file).getName(),
     55                    containerService.getKey(file),
     56                    null, // ifModifiedSince
     57                    null, // ifUnmodifiedSince
     58                    null, // ifMatch
     59                    null, // ifNoneMatch
     60                    status.isAppend() ? range.getStart() : null,
     61                    status.isAppend() ? (range.getEnd() == -1 ? null : range.getEnd()) : null);
    8362            if(log.isDebugEnabled()) {
    8463                log.debug(String.format("Reading stream with content length %d", object.getContentLength()));
  • trunk/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java

    r42090 r42426  
    7878    private static final Logger log = Logger.getLogger(S3Session.class);
    7979
    80     private DistributionConfiguration cdn;
    81 
    82     private Versioning versioning;
    83 
    8480    private final Preferences preferences
    8581            = PreferencesFactory.get();
     82
     83    private DistributionConfiguration cdn
     84            = new WebsiteCloudFrontDistributionConfiguration(this, trust, key);
     85
     86    private Versioning versioning
     87            = new S3VersioningFeature(this, new S3AccessControlListFeature(this));
    8688
    8789    private S3Protocol.AuthenticationHeaderSignatureVersion authenticationHeaderSignatureVersion
     
    244246        }
    245247        else {
    246             final AttributedList<Path> objects = new S3ObjectListService(this).list(directory, listener);
     248            AttributedList<Path> objects;
    247249            try {
    248                 objects.addAll(new S3VersionedObjectListService(this).list(directory, listener));
     250                objects = new S3VersionedObjectListService(this).list(directory, listener);
    249251            }
    250252            catch(AccessDeniedException | InteroperabilityException e) {
    251253                log.warn(String.format("Ignore failure listing versioned objects. %s", e.getDetail()));
     254                objects = new S3ObjectListService(this).list(directory, listener);
    252255            }
    253256            try {
     
    270273    public <T> T _getFeature(final Class<T> type) {
    271274        if(type == Read.class) {
    272             if(host.getHostname().endsWith(preferences.getProperty("s3.hostname.default"))) {
    273                 return (T) new S3ReadFeature(this);
    274             }
    275275            return (T) new S3ReadFeature(this);
    276276        }
     
    282282        }
    283283        if(type == Write.class) {
    284             if(host.getHostname().endsWith(preferences.getProperty("s3.hostname.default"))) {
    285                 return (T) new S3WriteFeature(this);
    286             }
    287284            return (T) new S3WriteFeature(this);
    288285        }
     
    331328        }
    332329        if(type == Versioning.class) {
    333             if(preferences.getBoolean("s3.revisions.enable")) {
    334                 if(null == versioning) {
    335                     versioning = new S3VersioningFeature(this, new S3AccessControlListFeature(this));
    336                 }
    337                 return (T) versioning;
    338             }
    339             return null;
     330            return (T) versioning;
    340331        }
    341332        if(type == Logging.class) {
     
    363354        }
    364355        if(type == DistributionConfiguration.class) {
    365             if(null == cdn) {
    366                 cdn = new WebsiteCloudFrontDistributionConfiguration(this, trust, key);
    367             }
    368356            return (T) cdn;
    369357        }
     
    382370                return (T) new S3TransferAccelerationService(this);
    383371            }
     372            return null;
    384373        }
    385374        if(type == Bulk.class) {
     
    388377                return (T) new S3BulkTransferAccelerationFeature(this, new S3TransferAccelerationService(this));
    389378            }
     379            return null;
    390380        }
    391381        if(type == Search.class) {
  • trunk/s3/src/main/java/ch/cyberduck/core/s3/S3TouchFeature.java

    r41833 r42426  
    2323import ch.cyberduck.core.MimeTypeService;
    2424import ch.cyberduck.core.Path;
     25import ch.cyberduck.core.PathAttributes;
    2526import ch.cyberduck.core.exception.BackgroundException;
    2627import ch.cyberduck.core.features.Encryption;
     
    3435
    3536import org.apache.commons.io.input.NullInputStream;
     37import org.jets3t.service.model.S3Object;
    3638import org.jets3t.service.model.StorageObject;
    3739
     
    7173        final StatusOutputStream<StorageObject> out = writer.write(file, status, new DisabledConnectionCallback());
    7274        new DefaultStreamCloser().close(out);
    73         return file;
     75        return new Path(file.getParent(), file.getName(), file.getType(),
     76                new PathAttributes(file.attributes()).withVersionId(((S3Object) out.getStatus()).getVersionId()));
    7477    }
    7578
  • trunk/s3/src/main/java/ch/cyberduck/core/s3/S3VersionedObjectListService.java

    r41507 r42426  
    2020import ch.cyberduck.core.ListService;
    2121import ch.cyberduck.core.Path;
     22import ch.cyberduck.core.PathAttributes;
    2223import ch.cyberduck.core.PathContainerService;
    2324import ch.cyberduck.core.PathNormalizer;
    2425import ch.cyberduck.core.exception.BackgroundException;
    25 import ch.cyberduck.core.features.Versioning;
    2626import ch.cyberduck.core.preferences.Preferences;
    2727import ch.cyberduck.core.preferences.PreferencesFactory;
    2828
     29import org.apache.commons.lang3.StringUtils;
     30import org.apache.log4j.Logger;
    2931import org.jets3t.service.ServiceException;
    3032import org.jets3t.service.VersionOrDeleteMarkersChunk;
     
    3739
    3840public class S3VersionedObjectListService implements ListService {
     41    private static final Logger log = Logger.getLogger(S3VersionedObjectListService.class);
    3942
    4043    private final Preferences preferences
     
    5255    @Override
    5356    public AttributedList<Path> list(final Path directory, final ListProgressListener listener) throws BackgroundException {
    54         final Versioning versioning = session.getFeature(Versioning.class);
    55         if(null == versioning) {
    56             return AttributedList.emptyList();
    57         }
    58         final S3ObjectListService list = new S3ObjectListService(session);
    59         final String prefix = list.createPrefix(directory);
     57        final String prefix = this.createPrefix(directory);
    6058        final Path bucket = containerService.getContainer(directory);
    6159        final AttributedList<Path> children = new AttributedList<Path>();
    6260        try {
    63             if(versioning.getConfiguration(bucket).isEnabled()) {
    64                 String priorLastKey = null;
    65                 String priorLastVersionId = null;
    66                 do {
    67                     final VersionOrDeleteMarkersChunk chunk = session.getClient().listVersionedObjectsChunked(
    68                             bucket.getName(), prefix, String.valueOf(Path.DELIMITER),
    69                             preferences.getInteger("s3.listing.chunksize"),
    70                             priorLastKey, priorLastVersionId, true);
    71                     // Amazon S3 returns object versions in the order in which they were
    72                     // stored, with the most recently stored returned first.
    73                     final List<BaseVersionOrDeleteMarker> items = Arrays.asList(chunk.getItems());
    74                     int i = 0;
    75                     for(BaseVersionOrDeleteMarker marker : items) {
    76                         if((marker.isDeleteMarker() && marker.isLatest()) || !marker.isLatest()) {
    77                             // Latest version already in default listing
    78                             final String key = PathNormalizer.normalize(marker.getKey());
    79                             if(new Path(bucket, key, EnumSet.of(Path.Type.directory)).equals(directory)) {
    80                                 continue;
    81                             }
    82                             final Path p = new Path(directory, PathNormalizer.name(key), EnumSet.of(Path.Type.file));
    83                             // Versioning is enabled if non null.
    84                             p.attributes().setVersionId(marker.getVersionId());
    85                             p.attributes().setRevision(++i);
    86                             p.attributes().setDuplicate(true);
    87                             p.attributes().setModificationDate(marker.getLastModified().getTime());
    88                             p.attributes().setRegion(bucket.attributes().getRegion());
    89                             if(marker instanceof S3Version) {
    90                                 p.attributes().setSize(((S3Version) marker).getSize());
    91                                 p.attributes().setETag(((S3Version) marker).getEtag());
    92                                 p.attributes().setStorageClass(((S3Version) marker).getStorageClass());
    93                             }
    94                             children.add(p);
    95                         }
     61            String priorLastKey = null;
     62            String priorLastVersionId = null;
     63            do {
     64                final VersionOrDeleteMarkersChunk chunk = session.getClient().listVersionedObjectsChunked(
     65                        bucket.getName(), prefix, String.valueOf(Path.DELIMITER),
     66                        preferences.getInteger("s3.listing.chunksize"),
     67                        priorLastKey, priorLastVersionId, true);
     68                // Amazon S3 returns object versions in the order in which they were
     69                // stored, with the most recently stored returned first.
     70                final List<BaseVersionOrDeleteMarker> items = Arrays.asList(chunk.getItems());
     71                int i = 0;
     72                for(BaseVersionOrDeleteMarker marker : items) {
     73                    final String key = PathNormalizer.normalize(marker.getKey());
     74                    if(new Path(bucket, key, EnumSet.of(Path.Type.directory)).equals(directory)) {
     75                        continue;
    9676                    }
    97                     priorLastKey = chunk.getNextKeyMarker();
    98                     priorLastVersionId = chunk.getNextVersionIdMarker();
    99                     listener.chunk(directory, children);
     77                    final Path p = new Path(directory, PathNormalizer.name(key), EnumSet.of(Path.Type.file));
     78                    // Versioning is enabled if non null.
     79                    p.attributes().setVersionId(marker.getVersionId());
     80                    p.attributes().setRevision(++i);
     81                    p.attributes().setDuplicate((marker.isDeleteMarker() && marker.isLatest()) || !marker.isLatest());
     82                    p.attributes().setModificationDate(marker.getLastModified().getTime());
     83                    p.attributes().setRegion(bucket.attributes().getRegion());
     84                    if(marker instanceof S3Version) {
     85                        p.attributes().setSize(((S3Version) marker).getSize());
     86                        p.attributes().setETag(((S3Version) marker).getEtag());
     87                        p.attributes().setStorageClass(((S3Version) marker).getStorageClass());
     88                    }
     89                    children.add(p);
    10090                }
    101                 while(priorLastKey != null);
     91                final String[] prefixes = chunk.getCommonPrefixes();
     92                for(String common : prefixes) {
     93                    if(common.equals(String.valueOf(Path.DELIMITER))) {
     94                        log.warn(String.format("Skipping prefix %s", common));
     95                        continue;
     96                    }
     97                    final String key = PathNormalizer.normalize(common);
     98                    if(new Path(bucket, key, EnumSet.of(Path.Type.directory)).equals(directory)) {
     99                        continue;
     100                    }
     101                    final PathAttributes attributes = new PathAttributes();
     102                    attributes.setRegion(bucket.attributes().getRegion());
     103                    final Path file = new Path(String.format("%s%s%s", bucket.getAbsolute(), String.valueOf(Path.DELIMITER), key), EnumSet.of(Path.Type.directory, Path.Type.placeholder), attributes);
     104                    children.add(file);
     105                }
     106                priorLastKey = chunk.getNextKeyMarker();
     107                priorLastVersionId = chunk.getNextVersionIdMarker();
     108                listener.chunk(directory, children);
    102109            }
     110            while(priorLastKey != null);
    103111            return children;
    104112        }
     
    107115        }
    108116    }
     117
     118    protected String createPrefix(final Path directory) {
     119        // Keys can be listed by prefix. By choosing a common prefix
     120        // for the names of related keys and marking these keys with
     121        // a special character that delimits hierarchy, you can use the list
     122        // operation to select and browse keys hierarchically
     123        String prefix = StringUtils.EMPTY;
     124        if(!containerService.isContainer(directory)) {
     125            // Restricts the response to only contain results that begin with the
     126            // specified prefix. If you omit this optional argument, the value
     127            // of Prefix for your query will be the empty string.
     128            // In other words, the results will be not be restricted by prefix.
     129            prefix = containerService.getKey(directory);
     130            if(!prefix.endsWith(String.valueOf(Path.DELIMITER))) {
     131                prefix += Path.DELIMITER;
     132            }
     133        }
     134        return prefix;
     135    }
    109136}
  • trunk/s3/src/test/java/ch/cyberduck/core/cryptomator/S3MultipartUploadServiceTest.java

    r42105 r42426  
    2121import ch.cyberduck.core.DisabledConnectionCallback;
    2222import ch.cyberduck.core.DisabledHostKeyCallback;
    23 import ch.cyberduck.core.DisabledListProgressListener;
    2423import ch.cyberduck.core.DisabledLoginCallback;
    2524import ch.cyberduck.core.DisabledPasswordCallback;
     
    2928import ch.cyberduck.core.Path;
    3029import ch.cyberduck.core.PathCache;
     30import ch.cyberduck.core.cryptomator.features.CryptoAttributesFeature;
    3131import ch.cyberduck.core.cryptomator.features.CryptoBulkFeature;
    3232import ch.cyberduck.core.cryptomator.features.CryptoDeleteFeature;
    3333import ch.cyberduck.core.cryptomator.features.CryptoFindFeature;
    34 import ch.cyberduck.core.cryptomator.features.CryptoListService;
    3534import ch.cyberduck.core.cryptomator.features.CryptoReadFeature;
    3635import ch.cyberduck.core.cryptomator.features.CryptoUploadFeature;
     
    3938import ch.cyberduck.core.io.DisabledStreamListener;
    4039import ch.cyberduck.core.io.StreamCopier;
     40import ch.cyberduck.core.s3.S3AttributesFinderFeature;
    4141import ch.cyberduck.core.s3.S3DefaultDeleteFeature;
    4242import ch.cyberduck.core.s3.S3FindFeature;
     
    105105        assertTrue(writeStatus.isComplete());
    106106        assertTrue(new CryptoFindFeature(session, new S3FindFeature(session), cryptomator).find(test));
    107         assertEquals(content.length, new CryptoListService(session, session, cryptomator).list(test.getParent(), new DisabledListProgressListener()).get(test).attributes().getSize());
     107        assertEquals(content.length, new CryptoAttributesFeature(session, new S3AttributesFinderFeature(session), cryptomator).find(test).getSize());
    108108        final ByteArrayOutputStream buffer = new ByteArrayOutputStream(content.length);
    109109        final TransferStatus readStatus = new TransferStatus().length(content.length);
     
    148148        assertTrue(writeStatus.isComplete());
    149149        assertTrue(new CryptoFindFeature(session, new S3FindFeature(session), cryptomator).find(test));
    150         assertEquals(content.length, new CryptoListService(session, session, cryptomator).list(test.getParent(), new DisabledListProgressListener()).get(test).attributes().getSize());
     150        assertEquals(content.length, new CryptoAttributesFeature(session, new S3AttributesFinderFeature(session), cryptomator).find(test).getSize());
    151151        final ByteArrayOutputStream buffer = new ByteArrayOutputStream(content.length);
    152152        final TransferStatus readStatus = new TransferStatus().length(content.length);
     
    190190        assertTrue(writeStatus.isComplete());
    191191        assertTrue(new CryptoFindFeature(session, new S3FindFeature(session), cryptomator).find(test));
    192         assertEquals(content.length, new CryptoListService(session, session, cryptomator).list(test.getParent(), new DisabledListProgressListener()).get(test).attributes().getSize());
     192        assertEquals(content.length, new CryptoAttributesFeature(session, new S3AttributesFinderFeature(session), cryptomator).find(test).getSize());
    193193        final ByteArrayOutputStream buffer = new ByteArrayOutputStream(content.length);
    194194        final TransferStatus readStatus = new TransferStatus().length(content.length);
  • trunk/s3/src/test/java/ch/cyberduck/core/cryptomator/S3WriteFeatureTest.java

    r42105 r42426  
    2121import ch.cyberduck.core.DisabledConnectionCallback;
    2222import ch.cyberduck.core.DisabledHostKeyCallback;
    23 import ch.cyberduck.core.DisabledListProgressListener;
    2423import ch.cyberduck.core.DisabledLoginCallback;
    2524import ch.cyberduck.core.DisabledPasswordCallback;
     
    2827import ch.cyberduck.core.Path;
    2928import ch.cyberduck.core.PathCache;
     29import ch.cyberduck.core.cryptomator.features.CryptoAttributesFeature;
    3030import ch.cyberduck.core.cryptomator.features.CryptoDeleteFeature;
    3131import ch.cyberduck.core.cryptomator.features.CryptoFindFeature;
    32 import ch.cyberduck.core.cryptomator.features.CryptoListService;
    3332import ch.cyberduck.core.cryptomator.features.CryptoReadFeature;
    3433import ch.cyberduck.core.cryptomator.features.CryptoWriteFeature;
     
    3635import ch.cyberduck.core.features.Delete;
    3736import ch.cyberduck.core.io.StreamCopier;
     37import ch.cyberduck.core.s3.S3AttributesFinderFeature;
    3838import ch.cyberduck.core.s3.S3DefaultDeleteFeature;
    3939import ch.cyberduck.core.s3.S3FindFeature;
     
    9595        out.close();
    9696        assertTrue(new CryptoFindFeature(session, new S3FindFeature(session), cryptomator).find(test));
    97         assertEquals(content.length, new CryptoListService(session, session, cryptomator).list(test.getParent(), new DisabledListProgressListener()).get(test).attributes().getSize());
     97        assertEquals(content.length, new CryptoAttributesFeature(session, new S3AttributesFinderFeature(session), cryptomator).find(test).getSize());
    9898        assertEquals(content.length, writer.append(test, status.getLength(), PathCache.empty()).size, 0L);
    9999        final ByteArrayOutputStream buffer = new ByteArrayOutputStream(content.length);
  • trunk/s3/src/test/java/ch/cyberduck/core/s3/S3AttributesFinderFeatureTest.java

    r42153 r42426  
    111111        final PathAttributes attributes = new S3AttributesFinderFeature(session).find(container);
    112112        assertEquals(-1L, attributes.getSize());
     113        assertNotNull(attributes.getRegion());
    113114        assertEquals(EnumSet.of(Path.Type.directory, Path.Type.volume), container.getType());
    114115        session.close();
  • trunk/s3/src/test/java/ch/cyberduck/core/s3/S3MultipartUploadServiceTest.java

    r42105 r42426  
    55import ch.cyberduck.core.DisabledConnectionCallback;
    66import ch.cyberduck.core.DisabledHostKeyCallback;
    7 import ch.cyberduck.core.DisabledListProgressListener;
    87import ch.cyberduck.core.DisabledLoginCallback;
    98import ch.cyberduck.core.DisabledPasswordStore;
     
    160159        assertTrue(status.isComplete());
    161160        assertTrue(new S3FindFeature(session).find(test));
    162         assertEquals(content.length, session.list(container,
    163                 new DisabledListProgressListener()).get(test).attributes().getSize());
     161        assertEquals(content.length, new S3AttributesFinderFeature(session).find(test).getSize());
    164162        new S3DefaultDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback());
    165163        local.delete();
     
    191189        assertTrue(status.isComplete());
    192190        assertTrue(new S3FindFeature(session).find(test));
    193         assertEquals(content.length, session.list(container,
    194                 new DisabledListProgressListener()).get(test).attributes().getSize());
     191        assertEquals(content.length, new S3AttributesFinderFeature(session).find(test).getSize());
    195192        new S3DefaultDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback());
    196193        local.delete();
     
    330327        assertTrue(append.isComplete());
    331328        assertTrue(new S3FindFeature(session).find(test));
    332         assertEquals(content.length, session.list(container,
    333                 new DisabledListProgressListener()).get(test).attributes().getSize());
     329        assertEquals(content.length, new S3AttributesFinderFeature(session).find(test).getSize());
    334330        final byte[] buffer = new byte[content.length];
    335331        final InputStream in = new S3ReadFeature(session).read(test, new TransferStatus(), new DisabledConnectionCallback());
  • trunk/s3/src/test/java/ch/cyberduck/core/s3/S3StorageClassFeatureTest.java

    r41062 r42426  
    2121import ch.cyberduck.core.DisabledCancelCallback;
    2222import ch.cyberduck.core.DisabledHostKeyCallback;
    23 import ch.cyberduck.core.DisabledListProgressListener;
    2423import ch.cyberduck.core.DisabledLoginCallback;
    2524import ch.cyberduck.core.DisabledPasswordStore;
     
    7776        session.login(new DisabledPasswordStore(), new DisabledLoginCallback(), new DisabledCancelCallback(), PathCache.empty());
    7877        final Path container = new Path("test-us-east-1-cyberduck", EnumSet.of(Path.Type.volume));
    79         final Path test = new Path(container, UUID.randomUUID().toString(), EnumSet.of(Path.Type.file));
    80         new S3TouchFeature(session).touch(test, new TransferStatus());
     78        final Path test = new S3TouchFeature(session).touch(new Path(container, UUID.randomUUID().toString(), EnumSet.of(Path.Type.file)), new TransferStatus());
    8179        final S3StorageClassFeature feature = new S3StorageClassFeature(session);
    8280        assertEquals(S3Object.STORAGE_CLASS_STANDARD, feature.getClass(test));
     
    8583        feature.setClass(test, S3Object.STORAGE_CLASS_REDUCED_REDUNDANCY);
    8684        assertEquals(S3Object.STORAGE_CLASS_REDUCED_REDUNDANCY, feature.getClass(test));
    87         assertEquals(S3Object.STORAGE_CLASS_REDUCED_REDUNDANCY, session.list(container,
    88                 new DisabledListProgressListener()).get(test).attributes().getStorageClass());
     85        assertEquals(S3Object.STORAGE_CLASS_REDUCED_REDUNDANCY, new S3AttributesFinderFeature(session).find(test).getStorageClass());
    8986        new S3DefaultDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback());
    9087        session.close();
  • trunk/s3/src/test/java/ch/cyberduck/core/s3/S3TouchFeatureTest.java

    r37613 r42426  
    11package ch.cyberduck.core.s3;
    22
     3import ch.cyberduck.core.AsciiRandomStringService;
    34import ch.cyberduck.core.Credentials;
    45import ch.cyberduck.core.DisabledCancelCallback;
     
    89import ch.cyberduck.core.Host;
    910import ch.cyberduck.core.Path;
     11import ch.cyberduck.core.PathAttributes;
    1012import ch.cyberduck.core.PathCache;
    1113import ch.cyberduck.core.exception.AccessDeniedException;
    1214import ch.cyberduck.core.features.Delete;
    1315import ch.cyberduck.core.features.Encryption;
     16import ch.cyberduck.core.shared.DefaultFindFeature;
    1417import ch.cyberduck.core.transfer.TransferStatus;
    1518import ch.cyberduck.test.IntegrationTest;
     
    1821import org.junit.experimental.categories.Category;
    1922
    20 import java.util.ArrayList;
    2123import java.util.Collections;
    2224import java.util.EnumSet;
    23 import java.util.List;
    2425import java.util.Map;
    2526import java.util.UUID;
     
    4647        session.login(new DisabledPasswordStore(), new DisabledLoginCallback(), new DisabledCancelCallback(), PathCache.empty());
    4748        final Path container = new Path("test-us-east-1-cyberduck", EnumSet.of(Path.Type.volume));
    48         final Path test = new Path(container, UUID.randomUUID().toString() + ".txt", EnumSet.of(Path.Type.file));
    49         new S3TouchFeature(session).touch(test, new TransferStatus());
     49        final Path test = new Path(container, new AsciiRandomStringService().random() + ".txt", EnumSet.of(Path.Type.file));
     50        assertNull(new S3TouchFeature(session).touch(test, new TransferStatus()).attributes().getVersionId());
    5051        assertTrue(new S3FindFeature(session).find(test));
    5152        final Map<String, String> metadata = new S3MetadataFeature(session, new S3AccessControlListFeature(session)).getMetadata(test);
    5253        assertFalse(metadata.isEmpty());
    5354        assertEquals("text/plain", metadata.get("Content-Type"));
    54         new S3DefaultDeleteFeature(session).delete(Collections.<Path>singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback());
     55        new S3DefaultDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback());
    5556        assertFalse(new S3FindFeature(session).find(test));
     57        session.close();
     58    }
     59
     60    @Test
     61    public void testTouchVersioning() throws Exception {
     62        final Host host = new Host(new S3Protocol(), new S3Protocol().getDefaultHostname(), new Credentials(
     63                System.getProperties().getProperty("s3.key"), System.getProperties().getProperty("s3.secret")
     64        ));
     65        final S3Session session = new S3Session(host);
     66        session.open(new DisabledHostKeyCallback());
     67        session.login(new DisabledPasswordStore(), new DisabledLoginCallback(), new DisabledCancelCallback(), PathCache.empty());
     68        final Path container = new Path("versioning-test-us-east-1-cyberduck", EnumSet.of(Path.Type.volume));
     69        final Path file = new Path(container, new AsciiRandomStringService().random(), EnumSet.of(Path.Type.file));
     70        final String version1 = new S3TouchFeature(session).touch(file, new TransferStatus()).attributes().getVersionId();
     71        final String version2 = new S3TouchFeature(session).touch(file, new TransferStatus()).attributes().getVersionId();
     72        assertTrue(new S3FindFeature(session).find(file));
     73        assertTrue(new DefaultFindFeature(session).find(file));
     74        assertTrue(new DefaultFindFeature(session).find(new Path(file.getParent(), file.getName(), file.getType(),
     75                new PathAttributes(file.attributes()).withVersionId(version1))));
     76        assertTrue(new DefaultFindFeature(session).find(new Path(file.getParent(), file.getName(), file.getType(),
     77                new PathAttributes(file.attributes()).withVersionId(version2))));
     78        assertTrue(new S3FindFeature(session).find(new Path(file.getParent(), file.getName(), file.getType(),
     79                new PathAttributes(file.attributes()).withVersionId(version1))));
     80        assertTrue(new S3FindFeature(session).find(new Path(file.getParent(), file.getName(), file.getType(),
     81                new PathAttributes(file.attributes()).withVersionId(version2))));
     82        new S3DefaultDeleteFeature(session).delete(Collections.singletonList(file), new DisabledLoginCallback(), new Delete.DisabledCallback());
     83        // Versioned files are not deleted but with delete marker added
     84        assertTrue(new DefaultFindFeature(session).find(new Path(file.getParent(), file.getName(), file.getType(),
     85                new PathAttributes(file.attributes()).withVersionId(version1))));
     86        assertTrue(new DefaultFindFeature(session).find(new Path(file.getParent(), file.getName(), file.getType(),
     87                new PathAttributes(file.attributes()).withVersionId(version2))));
     88        assertTrue((new S3FindFeature(session).find(new Path(file.getParent(), file.getName(), file.getType(),
     89                new PathAttributes(file.attributes()).withVersionId(version1)))));
     90        assertTrue((new S3FindFeature(session).find(new Path(file.getParent(), file.getName(), file.getType(),
     91                new PathAttributes(file.attributes()).withVersionId(version2)))));
    5692        session.close();
    5793    }
     
    101137        touch.touch(test, status);
    102138    }
    103 
    104     @Test
    105     public void testConnectionReuse() throws Exception {
    106         final S3Session session = new S3Session(
    107                 new Host(new S3Protocol(), new S3Protocol().getDefaultHostname(),
    108                         new Credentials(
    109                                 System.getProperties().getProperty("s3.key"), System.getProperties().getProperty("s3.secret")
    110                         )));
    111         session.open(new DisabledHostKeyCallback());
    112         session.login(new DisabledPasswordStore(), new DisabledLoginCallback(), new DisabledCancelCallback(), PathCache.empty());
    113         final S3TouchFeature service = new S3TouchFeature(session);
    114         final Path container = new Path("test-us-east-1-cyberduck", EnumSet.of(Path.Type.directory, Path.Type.volume));
    115         final List<Path> list = new ArrayList<Path>();
    116         for(int i = 0; i < 200; i++) {
    117             final String name = String.format("%s-%d", UUID.randomUUID().toString(), i);
    118             final Path test = new Path(container, name, EnumSet.of(Path.Type.file));
    119             service.touch(test, new TransferStatus());
    120             list.add(test);
    121         }
    122         new S3MultipleDeleteFeature(session).delete(list, new DisabledLoginCallback(), new Delete.DisabledCallback());
    123         session.close();
    124     }
    125139}
Note: See TracChangeset for help on using the changeset viewer.
swiss made software