From 32c7b2a2a923f4f446b28c324f867cbb711598a2 Mon Sep 17 00:00:00 2001 From: shs96c Date: Fri, 9 Feb 2018 22:43:49 +0300 Subject: [PATCH 01/15] Move NewSessionPayload so it can be used by ProtocolHandshake There's a lot of overlap between the ProtocolHandshake and the NewSessionPayload. Time to deal with this overlap between the two. The main change is to make the NewSessionPayload capable of writing to an Appendable. It does so by streaming the Capabilities used to create the payload as the OSS capabilities and those used by old geckodrivers, and then streaming every capability through transforms to generate spec compliant capabilities. There appears to be a bug where we always generate synthetic capabilities, but we can deal with that later. --- .../selenium/remote/NewSessionPayload.java | 638 ++++++++++++++++++ 1 file changed, 638 insertions(+) create mode 100644 src/main/java/org/openqa/selenium/remote/NewSessionPayload.java diff --git a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java new file mode 100644 index 000000000..2230b203f --- /dev/null +++ b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java @@ -0,0 +1,638 @@ +// Licensed to the Software Freedom Conservancy (SFC) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The SFC licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.openqa.selenium.remote; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; +import static org.openqa.selenium.json.Json.MAP_TYPE; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedMap; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Ordering; +import com.google.common.collect.Sets; +import com.google.common.io.CharStreams; + +import org.openqa.selenium.Capabilities; +import org.openqa.selenium.ImmutableCapabilities; +import org.openqa.selenium.io.FileHandler; +import org.openqa.selenium.json.Json; +import org.openqa.selenium.json.JsonInput; +import org.openqa.selenium.json.JsonOutput; +import org.openqa.selenium.remote.session.CapabilitiesFilter; +import org.openqa.selenium.remote.session.CapabilityTransform; +import org.openqa.selenium.remote.session.ChromeFilter; +import org.openqa.selenium.remote.session.EdgeFilter; +import org.openqa.selenium.remote.session.FirefoxFilter; +import org.openqa.selenium.remote.session.InternetExplorerFilter; +import org.openqa.selenium.remote.session.OperaFilter; +import org.openqa.selenium.remote.session.ProxyTransform; +import org.openqa.selenium.remote.session.SafariFilter; +import org.openqa.selenium.remote.session.StripAnyPlatform; +import org.openqa.selenium.remote.session.W3CNameTransform; +import org.openqa.selenium.remote.session.W3CPlatformNameNormaliser; + +import java.io.ByteArrayInputStream; +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.io.UncheckedIOException; +import java.io.Writer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Queue; +import java.util.ServiceLoader; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.logging.Logger; +import java.util.regex.Pattern; +import java.util.stream.Stream; + + +public class NewSessionPayload implements Closeable { + + private static final Logger LOG = Logger.getLogger(NewSessionPayload.class.getName()); + + private final Set adapters; + private final Set transforms; + + private static final Dialect DEFAULT_DIALECT = Dialect.OSS; + private final static Predicate ACCEPTED_W3C_PATTERNS = Stream.of( + "^[\\w-]+:.*$", + "^acceptInsecureCerts$", + "^browserName$", + "^browserVersion$", + "^platformName$", + "^pageLoadStrategy$", + "^proxy$", + "^setWindowRect$", + "^timeouts$", + "^unhandledPromptBehavior$") + .map(Pattern::compile) + .map(Pattern::asPredicate) + .reduce(identity -> false, Predicate::or); + + // Dedicate up to 10% of max ram to holding the payload + private static final long THRESHOLD = Runtime.getRuntime().maxMemory() / 10; + + private final Json json = new Json(); + private final Path root; + private final Sources sources; + + public static NewSessionPayload create(Capabilities caps) throws IOException { + // We need to convert the capabilities into a new session payload. At this point we're dealing + // with references, so I'm Just Sure This Will Be Fine. + + ImmutableMap.Builder builder = ImmutableMap.builder(); + + // OSS + builder.put("desiredCapabilities", caps.asMap()); + + // W3C Spec. + // TODO(simons): There's some serious overlap between ProtocolHandshake and this class. + ImmutableMap.Builder w3cCaps = ImmutableMap.builder(); + caps.asMap().entrySet().stream() + .filter(e -> ACCEPTED_W3C_PATTERNS.test(e.getKey())) + .filter(e -> e.getValue() != null) + .forEach(e -> w3cCaps.put(e.getKey(), e.getValue())); + builder.put( + "capabilities", ImmutableMap.of( + "firstMatch", ImmutableList.of(w3cCaps.build()))); + + return create(builder.build()); + } + + public static NewSessionPayload create(Map source) throws IOException { + Objects.requireNonNull(source, "Payload must be set"); + + byte[] json = new Json().toJson(source).getBytes(UTF_8); + return new NewSessionPayload( + json.length, + new InputStreamReader(new ByteArrayInputStream(json), UTF_8)); + } + + public static NewSessionPayload create(long size, Reader source) throws IOException { + return new NewSessionPayload(size, source); + } + + private NewSessionPayload(long size, Reader source) throws IOException { + Sources sources; + if (size > THRESHOLD || Runtime.getRuntime().freeMemory() < size) { + this.root = Files.createTempDirectory("new-session"); + sources = diskBackedSource(source); + } else { + this.root = null; + sources = memoryBackedSource(new Json().toType(CharStreams.toString(source), Map.class)); + } + + validate(sources); + this.sources = rewrite(sources); + + ImmutableSet.Builder adapters = ImmutableSet.builder(); + ServiceLoader.load(CapabilitiesFilter.class).forEach(adapters::add); + adapters + .add(new ChromeFilter()) + .add(new EdgeFilter()) + .add(new FirefoxFilter()) + .add(new InternetExplorerFilter()) + .add(new OperaFilter()) + .add(new SafariFilter()); + this.adapters = adapters.build(); + + ImmutableSet.Builder transforms = ImmutableSet.builder(); + ServiceLoader.load(CapabilityTransform.class).forEach(transforms::add); + transforms + .add(new ProxyTransform()) + .add(new StripAnyPlatform()) + .add(new W3CPlatformNameNormaliser()) + .add(new W3CNameTransform()); + this.transforms = transforms.build(); + } + + public void writeTo(Appendable appendable) throws IOException { + try (JsonOutput json = new Json().newOutput(appendable)) { + json.beginObject(); + + @SuppressWarnings("unchecked") + Map first = (Map) stream().findFirst() + .orElse(new ImmutableCapabilities()) + .asMap(); + + // Write the first capability we get as the desired capability. + json.name("desiredCapabilities"); + json.write(first, Json.MAP_TYPE); + + // And write the first capability for gecko13 + json.name("capabilities"); + json.beginObject(); + + json.name("desiredCapabilities"); + json.write(first, Json.MAP_TYPE); + + // Then write everything into the w3c payload. Because of the way we do this, it's easiest + // to just populate the "firstMatch" section. The spec says it's fine to omit the + // "alwaysMatch" field, so we do this. + json.name("firstMatch"); + json.beginArray(); + //noinspection unchecked + stream() + .map(Capabilities::asMap) + .map(map -> (Map) map) + .forEach(map -> streamW3CProtocolParameters(json, map)); + json.endArray(); + + json.endObject(); // Close "capabilities" object + json.endObject(); + } + } + + private void validate(Sources sources) { + if (!sources.getDialects().contains(Dialect.W3C)) { + return; // Nothing to do + } + + // Ensure that the W3C payload looks okay + Map alwaysMatch = sources.getAlwaysMatch().get(); + validateSpecCompliance(alwaysMatch); + + Set duplicateKeys = sources.getFirstMatch().stream() + .map(Supplier::get) + .peek(this::validateSpecCompliance) + .map(fragment -> Sets.intersection(alwaysMatch.keySet(), fragment.keySet())) + .flatMap(Collection::stream) + .collect(ImmutableSortedSet.toImmutableSortedSet(Ordering.natural())); + + if (!duplicateKeys.isEmpty()) { + throw new IllegalArgumentException( + "W3C payload contained keys duplicated between the firstMatch and alwaysMatch items: " + + duplicateKeys); + } + } + + private void validateSpecCompliance(Map fragment) { + ImmutableList badKeys = fragment.keySet().stream() + .filter(ACCEPTED_W3C_PATTERNS.negate()) + .collect(ImmutableList.toImmutableList()); + + if (!badKeys.isEmpty()) { + throw new IllegalArgumentException( + "W3C payload contained keys that do not comply with the spec: " + badKeys); + } + } + + private void streamW3CProtocolParameters(JsonOutput out, Map des) { + // Technically we should be building up a combination of "alwaysMatch" and "firstMatch" options. + // We're going to do a little processing to figure out what we might be able to do, and assume + // that people don't really understand the difference between required and desired (which is + // commonly the case). Wish us luck. Looking at the current implementations, people may have + // set options for multiple browsers, in which case a compliant W3C remote end won't start + // a session. If we find this, then we create multiple firstMatch capabilities. Furrfu. + // The table of options are: + // + // Chrome: chromeOptions + // Firefox: moz:.*, firefox_binary, firefox_profile, marionette + // Edge: none given + // IEDriver: ignoreZoomSetting, initialBrowserUrl, enableElementCacheCleanup, + // browserAttachTimeout, enablePersistentHover, requireWindowFocus, logFile, logLevel, host, + // extractPath, silent, ie.* + // Opera: operaOptions + // SafariDriver: safari.options + // + // We can't use the constants defined in the classes because it would introduce circular + // dependencies between the remote library and the implementations. Yay! + + ImmutableList> firstMatch = adapters.stream() + .map(adapter -> adapter.apply(des)) + .filter(Objects::nonNull) + .map(this::applyTransforms) + .filter(w3cCaps -> !w3cCaps.isEmpty()) + .collect(ImmutableList.toImmutableList()); + + Set excludedKeys = firstMatch.stream() + .map(Map::keySet) + .flatMap(Collection::stream) + .distinct() + .collect(ImmutableSet.toImmutableSet()); + + Map alwaysMatch = applyTransforms(des).entrySet().stream() + .filter(entry -> !excludedKeys.contains(entry.getKey())) + .filter(entry -> entry.getValue() != null) + .collect(ImmutableSortedMap.toImmutableSortedMap( + Ordering.natural(), + Map.Entry::getKey, + Map.Entry::getValue)); + + firstMatch.stream() + .map(first -> ImmutableSortedMap.naturalOrder().putAll(alwaysMatch).putAll(first).build()) + .forEach(map -> out.write(map, MAP_TYPE)); + } + + private Map applyTransforms(Map caps) { + Queue> toExamine = new LinkedList<>(); + toExamine.addAll(caps.entrySet()); + Set seenKeys = new HashSet<>(); + Map toReturn = new TreeMap<>(); + + // Take each entry and apply the transforms + while (!toExamine.isEmpty()) { + Map.Entry entry = toExamine.remove(); + seenKeys.add(entry.getKey()); + + if (entry.getValue() == null) { + continue; + } + + for (CapabilityTransform transform : transforms) { + Collection> result = transform.apply(entry); + if (result == null) { + toReturn.remove(entry.getKey()); + break; + } + + for (Map.Entry newEntry : result) { + if (!seenKeys.contains(newEntry.getKey())) { + toExamine.add(newEntry); + } else { + if (newEntry.getKey().equals(entry.getKey())) { + entry = newEntry; + } + toReturn.put(newEntry.getKey(), newEntry.getValue()); + } + } + } + } + + return toReturn; + } + + /** + * If the local end sent a request with a JSON Wire Protocol payload that does not have a matching + * W3C payload, then we need to synthesize one that matches. + */ + private Sources rewrite(Sources sources) { + if (!sources.getDialects().contains(Dialect.OSS)) { + // Yay! Nothing to do! + return sources; + } + + if (!sources.getDialects().contains(Dialect.W3C)) { + // Yay! Also nothing to do. I mean, we have an empty payload, but that's cool. + return sources; + } + + Map ossPayload = sources.getOss().get().entrySet().stream() + .filter(e -> !("platform".equals(e.getKey()) && "ANY".equals(e.getValue()))) + .filter(e -> !("version".equals(e.getKey()) && "".equals(e.getValue()))) + .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); + Map always = sources.getAlwaysMatch().get(); + Optional> w3cMatch = sources.getFirstMatch().stream() + .map(Supplier::get) + .map(m -> ImmutableMap.builder().putAll(always).putAll(m).build()) + .filter(m -> m.equals(ossPayload)) + .findAny(); + if (w3cMatch.isPresent()) { + // There's a w3c capability that matches the oss one. Nothing to do. + LOG.fine("Found a w3c capability that matches the oss one."); + return sources; + } + + LOG.info("Mismatched capabilities. Creating a synthetic w3c capability."); + + ImmutableList.Builder>> newFirstMatches = ImmutableList.builder(); + newFirstMatches.add(sources.getOss()); + sources.getFirstMatch() + .forEach(m -> newFirstMatches.add(() -> { + ImmutableMap.Builder builder = ImmutableMap.builder(); + builder.putAll(sources.getAlwaysMatch().get()); + builder.putAll(m.get()); + return builder.build(); + })); + + return new Sources( + sources.getOriginalPayload(), + sources.getPayloadSize(), + sources.getOss(), + ImmutableMap::of, + newFirstMatches.build(), + sources.getDialects()); + } + + private Sources memoryBackedSource(Map source) { + LOG.fine("Memory-based payload for: " + source); + + Set dialects = new TreeSet<>(); + Map oss = toMap(source.get("desiredCapabilities")); + if (oss != null) { + dialects.add(Dialect.OSS); + } + + Map alwaysMatch = new TreeMap<>(); + List>> firstMatches = new LinkedList<>(); + + Map caps = toMap(source.get("capabilities")); + if (caps != null) { + Map always = toMap(caps.get("alwaysMatch")); + if (always != null) { + alwaysMatch.putAll(always); + dialects.add(Dialect.W3C); + } + Object raw = caps.get("firstMatch"); + if (raw instanceof Collection) { + ((Collection) raw).stream() + .map(NewSessionPayload::toMap) + .filter(Objects::nonNull) + .forEach(m -> firstMatches.add(() -> m)); + dialects.add(Dialect.W3C); + } + if (firstMatches.isEmpty()) { + firstMatches.add(ImmutableMap::of); + } + } + + byte[] json = new Json().toJson(source).getBytes(UTF_8); + + return new Sources( + () -> new ByteArrayInputStream(json), + json.length, + () -> oss, + () -> alwaysMatch, + firstMatches, + dialects); + } + + private Sources diskBackedSource(Reader source) throws IOException { + LOG.fine("Disk-based payload for " + source); + + // Copy the original payload to disk + Path payload = root.resolve("original-payload.json"); + try (Writer out = Files.newBufferedWriter(payload, UTF_8)) { + CharStreams.copy(source, out); + } + + try ( + Reader in = Files.newBufferedReader(payload); + JsonInput jin = json.newInput(in)) { + Set dialects = new TreeSet<>(); + Supplier> oss = null; + Supplier> always = ImmutableMap::of; + List>> first = new LinkedList<>(); + + jin.beginObject(); + + while (jin.hasNext()) { + switch (jin.nextName()) { + case "capabilities": + jin.beginObject(); + while (jin.hasNext()) { + switch (jin.nextName()) { + + case "alwaysMatch": + Path a = write("always-match.json", jin); + always = () -> read(a); + dialects.add(Dialect.W3C); + break; + + case "firstMatch": + jin.beginArray(); + int i = 0; + while (jin.hasNext()) { + Path f = write("first-match-" + i + ".json", jin); + first.add(() -> read(f)); + i++; + } + jin.endArray(); + dialects.add(Dialect.W3C); + break; + + default: + jin.skipValue(); + } + } + jin.endObject(); + break; + + case "desiredCapabilities": + Path ossCaps = write("oss.json", jin); + oss = () -> read(ossCaps); + dialects.add(Dialect.OSS); + break; + + default: + jin.skipValue(); + } + } + jin.endObject(); + + if (first.isEmpty()) { + first.add(ImmutableMap::of); + } + + return new Sources( + () -> { + try { + return Files.newInputStream(payload); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }, + Files.size(payload), + oss, + always, + first, + dialects); + } finally { + source.close(); + } + } + + private Path write(String fileName, JsonInput json) { + Path out = root.resolve(fileName); + + try (Writer writer = Files.newBufferedWriter(out, UTF_8, TRUNCATE_EXISTING, CREATE); + JsonOutput jsonOut = this.json.newOutput(writer)) { + jsonOut.write(json, MAP_TYPE); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + return out; + } + + private Map read(Path path) { + try (Reader reader = Files.newBufferedReader(path, UTF_8); + JsonInput jin = json.newInput(reader)) { + return jin.read(MAP_TYPE); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static Map toMap(Object obj) { + if (!(obj instanceof Map)) { + return null; + } + + return ((Map) obj).entrySet() + .stream() + .filter(e -> e.getKey() != null) + .filter(e -> e.getValue() != null) + .collect(ImmutableMap.toImmutableMap(e -> String.valueOf(e.getKey()), Map.Entry::getValue)); + } + + public Stream stream() throws IOException { + Stream> mapStream; + + if (getDownstreamDialects().contains(Dialect.W3C)) { + Map always = sources.getAlwaysMatch().get(); + mapStream = sources.getFirstMatch().stream() + .map(Supplier::get) + .map(m -> ImmutableMap.builder().putAll(always).putAll(m).build()); + } else if (getDownstreamDialects().contains(Dialect.OSS)) { + mapStream = Stream.of(sources.getOss().get()); + } else { + mapStream = Stream.of(ImmutableMap.of()); + } + + return mapStream.map(ImmutableCapabilities::new); + } + + public ImmutableSet getDownstreamDialects() { + return sources.getDialects().isEmpty() ? + ImmutableSet.of(DEFAULT_DIALECT) : + sources.getDialects(); + } + + public Supplier getPayload() { + return sources.getOriginalPayload(); + } + + public long getPayloadSize() { + return sources.getPayloadSize(); + } + + @Override + public void close() { + if (root != null) { + FileHandler.delete(root.toAbsolutePath().toFile()); + } + } + + private static class Sources { + + private final Supplier originalPayload; + private final long payloadSizeInBytes; + private final Supplier> oss; + private final Supplier> alwaysMatch; + private final List>> firstMatch; + private final ImmutableSet dialects; + + Sources( + Supplier originalPayload, + long payloadSizeInBytes, + Supplier> oss, + Supplier> alwaysMatch, + List>> firstMatch, + Set dialects) { + this.originalPayload = originalPayload; + this.payloadSizeInBytes = payloadSizeInBytes; + this.oss = oss; + this.alwaysMatch = alwaysMatch; + this.firstMatch = firstMatch; + this.dialects = ImmutableSet.copyOf(dialects); + } + + Supplier getOriginalPayload() { + return originalPayload; + } + + Supplier> getOss() { + return oss; + } + + Supplier> getAlwaysMatch() { + return alwaysMatch; + } + + List>> getFirstMatch() { + return firstMatch; + } + + ImmutableSet getDialects() { + return dialects; + } + + public long getPayloadSize() { + return payloadSizeInBytes; + } + } +} \ No newline at end of file From 368773157023a9f6bf94c62f114320a52f2789fc Mon Sep 17 00:00:00 2001 From: shs96c Date: Fri, 9 Feb 2018 22:47:06 +0300 Subject: [PATCH 02/15] Better docs for NewSessionPayload.stream --- .../org/openqa/selenium/remote/NewSessionPayload.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java index 2230b203f..76d980f11 100644 --- a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java +++ b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java @@ -549,6 +549,16 @@ private static Map toMap(Object obj) { .collect(ImmutableMap.toImmutableMap(e -> String.valueOf(e.getKey()), Map.Entry::getValue)); } + /** + * Stream the {@link Capabilities} encoded in the payload used to create this instance. The + * {@link Stream} will start with a {@link Capabilities} object matching the OSS capabilities, and + * will then expand each of the "{@code firstMatch}" and "{@code alwaysMatch}" contents as defined + * in the W3C WebDriver spec. + *

+ * The OSS {@link Capabilities} are listed first because converting the OSS capabilities to the + * equivalent W3C capabilities isn't particularly easy, so it's hoped that this approach gives us + * the most compatible implementation. + */ public Stream stream() throws IOException { Stream> mapStream; From b67cfdce664cd8ae884de225c9fca25cdcbc4718 Mon Sep 17 00:00:00 2001 From: shs96c Date: Fri, 9 Feb 2018 22:48:43 +0300 Subject: [PATCH 03/15] Fix the Protocol Handshake --- .../selenium/remote/NewSessionPayload.java | 697 +++++++----------- 1 file changed, 272 insertions(+), 425 deletions(-) diff --git a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java index 76d980f11..3b6afdeff 100644 --- a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java +++ b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java @@ -18,22 +18,21 @@ package org.openqa.selenium.remote; import static java.nio.charset.StandardCharsets.UTF_8; -import static java.nio.file.StandardOpenOption.CREATE; -import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; import static org.openqa.selenium.json.Json.MAP_TYPE; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.ImmutableSortedMap; import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Ordering; import com.google.common.collect.Sets; +import com.google.common.io.CharSource; import com.google.common.io.CharStreams; +import com.google.common.io.FileBackedOutputStream; +import com.google.gson.reflect.TypeToken; import org.openqa.selenium.Capabilities; import org.openqa.selenium.ImmutableCapabilities; -import org.openqa.selenium.io.FileHandler; import org.openqa.selenium.json.Json; import org.openqa.selenium.json.JsonInput; import org.openqa.selenium.json.JsonOutput; @@ -47,35 +46,29 @@ import org.openqa.selenium.remote.session.ProxyTransform; import org.openqa.selenium.remote.session.SafariFilter; import org.openqa.selenium.remote.session.StripAnyPlatform; -import org.openqa.selenium.remote.session.W3CNameTransform; import org.openqa.selenium.remote.session.W3CPlatformNameNormaliser; -import java.io.ByteArrayInputStream; import java.io.Closeable; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; +import java.io.OutputStreamWriter; import java.io.Reader; -import java.io.UncheckedIOException; +import java.io.StringReader; import java.io.Writer; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Queue; import java.util.ServiceLoader; import java.util.Set; import java.util.TreeMap; -import java.util.TreeSet; import java.util.function.Predicate; -import java.util.function.Supplier; import java.util.logging.Logger; import java.util.regex.Pattern; +import java.util.stream.Collectors; import java.util.stream.Stream; @@ -102,62 +95,41 @@ public class NewSessionPayload implements Closeable { .map(Pattern::asPredicate) .reduce(identity -> false, Predicate::or); - // Dedicate up to 10% of max ram to holding the payload - private static final long THRESHOLD = Runtime.getRuntime().maxMemory() / 10; - private final Json json = new Json(); - private final Path root; - private final Sources sources; + private final FileBackedOutputStream backingStore; + private final ImmutableSet dialects; public static NewSessionPayload create(Capabilities caps) throws IOException { // We need to convert the capabilities into a new session payload. At this point we're dealing // with references, so I'm Just Sure This Will Be Fine. - - ImmutableMap.Builder builder = ImmutableMap.builder(); - - // OSS - builder.put("desiredCapabilities", caps.asMap()); - - // W3C Spec. - // TODO(simons): There's some serious overlap between ProtocolHandshake and this class. - ImmutableMap.Builder w3cCaps = ImmutableMap.builder(); - caps.asMap().entrySet().stream() - .filter(e -> ACCEPTED_W3C_PATTERNS.test(e.getKey())) - .filter(e -> e.getValue() != null) - .forEach(e -> w3cCaps.put(e.getKey(), e.getValue())); - builder.put( - "capabilities", ImmutableMap.of( - "firstMatch", ImmutableList.of(w3cCaps.build()))); - - return create(builder.build()); + return create(ImmutableMap.of("desiredCapabilities", caps.asMap())); } public static NewSessionPayload create(Map source) throws IOException { Objects.requireNonNull(source, "Payload must be set"); - byte[] json = new Json().toJson(source).getBytes(UTF_8); - return new NewSessionPayload( - json.length, - new InputStreamReader(new ByteArrayInputStream(json), UTF_8)); + String json = new Json().toJson(source); + return new NewSessionPayload(new StringReader(json)); } - public static NewSessionPayload create(long size, Reader source) throws IOException { - return new NewSessionPayload(size, source); + public static NewSessionPayload create(Reader source) throws IOException { + return new NewSessionPayload(source); } - private NewSessionPayload(long size, Reader source) throws IOException { - Sources sources; - if (size > THRESHOLD || Runtime.getRuntime().freeMemory() < size) { - this.root = Files.createTempDirectory("new-session"); - sources = diskBackedSource(source); - } else { - this.root = null; - sources = memoryBackedSource(new Json().toType(CharStreams.toString(source), Map.class)); + private NewSessionPayload(Reader source) throws IOException { + // Dedicate up to 10% of all RAM or 20% of available RAM (whichever is smaller) to storing this + // payload. + int threshold = (int) Math.min( + Integer.MAX_VALUE, + Math.min( + Runtime.getRuntime().freeMemory() / 5, + Runtime.getRuntime().maxMemory() / 10)); + + backingStore = new FileBackedOutputStream(threshold); + try (Writer writer = new OutputStreamWriter(backingStore, UTF_8)) { + CharStreams.copy(source, writer); } - validate(sources); - this.sources = rewrite(sources); - ImmutableSet.Builder adapters = ImmutableSet.builder(); ServiceLoader.load(CapabilitiesFilter.class).forEach(adapters::add); adapters @@ -174,9 +146,71 @@ private NewSessionPayload(long size, Reader source) throws IOException { transforms .add(new ProxyTransform()) .add(new StripAnyPlatform()) - .add(new W3CPlatformNameNormaliser()) - .add(new W3CNameTransform()); + .add(new W3CPlatformNameNormaliser()); this.transforms = transforms.build(); + + ImmutableSet.Builder dialects = ImmutableSet.builder(); + if (getOss() != null) { + dialects.add(Dialect.OSS); + } + if (getAlwaysMatch() != null || getFirstMatches() != null) { + dialects.add(Dialect.W3C); + } + this.dialects = dialects.build(); + + validate(); + } + + private void validate() throws IOException { + Map alwaysMatch = getAlwaysMatch(); + if (alwaysMatch == null) { + alwaysMatch = ImmutableMap.of(); + } + Map always = alwaysMatch; + Collection> firsts = getFirstMatches(); + if (firsts == null) { + firsts = ImmutableList.of(ImmutableMap.of()); + } + + if (firsts.isEmpty()) { + throw new IllegalArgumentException("First match w3c capabilities is zero length"); + } + + firsts.stream() + .peek(map -> { + Set overlap = Sets.intersection(always.keySet(), map.keySet()); + if (!overlap.isEmpty()) { + throw new IllegalArgumentException( + "Overlapping keys between w3c always and first match capabilities: " + overlap); + } + }) + .map(first -> { + Map toReturn = new HashMap<>(); + toReturn.putAll(always); + toReturn.putAll(first); + return toReturn; + }) + .peek(map -> { + ImmutableSortedSet nullKeys = map.entrySet().stream() + .filter(entry -> entry.getValue() == null) + .map(Map.Entry::getKey) + .collect(ImmutableSortedSet.toImmutableSortedSet(Ordering.natural())); + if (!nullKeys.isEmpty()) { + throw new IllegalArgumentException( + "Null values found in w3c capabilities. Keys are: " + nullKeys); + } + }) + .peek(map -> { + ImmutableSortedSet illegalKeys = map.entrySet().stream() + .filter(entry -> !ACCEPTED_W3C_PATTERNS.test(entry.getKey())) + .map(Map.Entry::getKey) + .collect(ImmutableSortedSet.toImmutableSortedSet(Ordering.natural())); + if (!illegalKeys.isEmpty()) { + throw new IllegalArgumentException( + "Illegal key values seen in w3c capabilities: " + illegalKeys); + } + }) + .forEach(map -> {}); } public void writeTo(Appendable appendable) throws IOException { @@ -190,14 +224,14 @@ public void writeTo(Appendable appendable) throws IOException { // Write the first capability we get as the desired capability. json.name("desiredCapabilities"); - json.write(first, Json.MAP_TYPE); + json.write(first, MAP_TYPE); // And write the first capability for gecko13 json.name("capabilities"); json.beginObject(); json.name("desiredCapabilities"); - json.write(first, Json.MAP_TYPE); + json.write(first, MAP_TYPE); // Then write everything into the w3c payload. Because of the way we do this, it's easiest // to just populate the "firstMatch" section. The spec says it's fine to omit the @@ -208,7 +242,7 @@ public void writeTo(Appendable appendable) throws IOException { stream() .map(Capabilities::asMap) .map(map -> (Map) map) - .forEach(map -> streamW3CProtocolParameters(json, map)); + .forEach(map -> json.write(map, MAP_TYPE)); json.endArray(); json.endObject(); // Close "capabilities" object @@ -216,40 +250,6 @@ public void writeTo(Appendable appendable) throws IOException { } } - private void validate(Sources sources) { - if (!sources.getDialects().contains(Dialect.W3C)) { - return; // Nothing to do - } - - // Ensure that the W3C payload looks okay - Map alwaysMatch = sources.getAlwaysMatch().get(); - validateSpecCompliance(alwaysMatch); - - Set duplicateKeys = sources.getFirstMatch().stream() - .map(Supplier::get) - .peek(this::validateSpecCompliance) - .map(fragment -> Sets.intersection(alwaysMatch.keySet(), fragment.keySet())) - .flatMap(Collection::stream) - .collect(ImmutableSortedSet.toImmutableSortedSet(Ordering.natural())); - - if (!duplicateKeys.isEmpty()) { - throw new IllegalArgumentException( - "W3C payload contained keys duplicated between the firstMatch and alwaysMatch items: " + - duplicateKeys); - } - } - - private void validateSpecCompliance(Map fragment) { - ImmutableList badKeys = fragment.keySet().stream() - .filter(ACCEPTED_W3C_PATTERNS.negate()) - .collect(ImmutableList.toImmutableList()); - - if (!badKeys.isEmpty()) { - throw new IllegalArgumentException( - "W3C payload contained keys that do not comply with the spec: " + badKeys); - } - } - private void streamW3CProtocolParameters(JsonOutput out, Map des) { // Technically we should be building up a combination of "alwaysMatch" and "firstMatch" options. // We're going to do a little processing to figure out what we might be able to do, and assume @@ -270,379 +270,226 @@ private void streamW3CProtocolParameters(JsonOutput out, Map des // // We can't use the constants defined in the classes because it would introduce circular // dependencies between the remote library and the implementations. Yay! + } - ImmutableList> firstMatch = adapters.stream() - .map(adapter -> adapter.apply(des)) + /** + * Stream the {@link Capabilities} encoded in the payload used to create this instance. The + * {@link Stream} will start with a {@link Capabilities} object matching the OSS capabilities, and + * will then expand each of the "{@code firstMatch}" and "{@code alwaysMatch}" contents as defined + * in the W3C WebDriver spec. + *

+ * The OSS {@link Capabilities} are listed first because converting the OSS capabilities to the + * equivalent W3C capabilities isn't particularly easy, so it's hoped that this approach gives us + * the most compatible implementation. + */ + public Stream stream() throws IOException { + // OSS first + Stream> oss = Stream.of(getOss()); + + // And now W3C + Stream> w3c = getW3C(); + + return Stream.concat(oss, w3c) .filter(Objects::nonNull) .map(this::applyTransforms) - .filter(w3cCaps -> !w3cCaps.isEmpty()) - .collect(ImmutableList.toImmutableList()); - - Set excludedKeys = firstMatch.stream() - .map(Map::keySet) - .flatMap(Collection::stream) + .filter(Objects::nonNull) .distinct() - .collect(ImmutableSet.toImmutableSet()); - - Map alwaysMatch = applyTransforms(des).entrySet().stream() - .filter(entry -> !excludedKeys.contains(entry.getKey())) - .filter(entry -> entry.getValue() != null) - .collect(ImmutableSortedMap.toImmutableSortedMap( - Ordering.natural(), - Map.Entry::getKey, - Map.Entry::getValue)); - - firstMatch.stream() - .map(first -> ImmutableSortedMap.naturalOrder().putAll(alwaysMatch).putAll(first).build()) - .forEach(map -> out.write(map, MAP_TYPE)); + .map(ImmutableCapabilities::new); } - private Map applyTransforms(Map caps) { - Queue> toExamine = new LinkedList<>(); - toExamine.addAll(caps.entrySet()); - Set seenKeys = new HashSet<>(); - Map toReturn = new TreeMap<>(); - - // Take each entry and apply the transforms - while (!toExamine.isEmpty()) { - Map.Entry entry = toExamine.remove(); - seenKeys.add(entry.getKey()); - - if (entry.getValue() == null) { - continue; - } + public ImmutableSet getDownstreamDialects() { + return dialects.isEmpty() ? ImmutableSet.of(Dialect.OSS) : dialects; + } - for (CapabilityTransform transform : transforms) { - Collection> result = transform.apply(entry); - if (result == null) { - toReturn.remove(entry.getKey()); - break; - } + @Override + public void close() throws IOException { + backingStore.reset(); + } - for (Map.Entry newEntry : result) { - if (!seenKeys.contains(newEntry.getKey())) { - toExamine.add(newEntry); - } else { - if (newEntry.getKey().equals(entry.getKey())) { - entry = newEntry; - } - toReturn.put(newEntry.getKey(), newEntry.getValue()); - } + private Map getOss() throws IOException { + CharSource charSource = backingStore.asByteSource().asCharSource(UTF_8); + try (Reader reader = charSource.openBufferedStream(); + JsonInput input = json.newInput(reader)) { + input.beginObject(); + while (input.hasNext()) { + String name = input.nextName(); + if ("desiredCapabilities".equals(name)) { + return input.read(MAP_TYPE); + } else { + input.skipValue(); } } } - - return toReturn; + return null; } - /** - * If the local end sent a request with a JSON Wire Protocol payload that does not have a matching - * W3C payload, then we need to synthesize one that matches. - */ - private Sources rewrite(Sources sources) { - if (!sources.getDialects().contains(Dialect.OSS)) { - // Yay! Nothing to do! - return sources; - } - - if (!sources.getDialects().contains(Dialect.W3C)) { - // Yay! Also nothing to do. I mean, we have an empty payload, but that's cool. - return sources; - } - - Map ossPayload = sources.getOss().get().entrySet().stream() - .filter(e -> !("platform".equals(e.getKey()) && "ANY".equals(e.getValue()))) - .filter(e -> !("version".equals(e.getKey()) && "".equals(e.getValue()))) - .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); - Map always = sources.getAlwaysMatch().get(); - Optional> w3cMatch = sources.getFirstMatch().stream() - .map(Supplier::get) - .map(m -> ImmutableMap.builder().putAll(always).putAll(m).build()) - .filter(m -> m.equals(ossPayload)) - .findAny(); - if (w3cMatch.isPresent()) { - // There's a w3c capability that matches the oss one. Nothing to do. - LOG.fine("Found a w3c capability that matches the oss one."); - return sources; - } - - LOG.info("Mismatched capabilities. Creating a synthetic w3c capability."); - - ImmutableList.Builder>> newFirstMatches = ImmutableList.builder(); - newFirstMatches.add(sources.getOss()); - sources.getFirstMatch() - .forEach(m -> newFirstMatches.add(() -> { - ImmutableMap.Builder builder = ImmutableMap.builder(); - builder.putAll(sources.getAlwaysMatch().get()); - builder.putAll(m.get()); - return builder.build(); - })); - - return new Sources( - sources.getOriginalPayload(), - sources.getPayloadSize(), - sources.getOss(), - ImmutableMap::of, - newFirstMatches.build(), - sources.getDialects()); - } - - private Sources memoryBackedSource(Map source) { - LOG.fine("Memory-based payload for: " + source); - - Set dialects = new TreeSet<>(); - Map oss = toMap(source.get("desiredCapabilities")); + private Stream> getW3C() throws IOException { + // If there's an OSS value, generate a stream of capabilities from that using the transforms, + // then add magic to generate each of the w3c capabilities. For the sake of simplicity, we're + // going to make the (probably wrong) assumption we can hold all of the firstMatch values and + // alwaysMatch value in memory at the same time. + Map oss = getOss(); + Stream> fromOss; if (oss != null) { - dialects.add(Dialect.OSS); - } - - Map alwaysMatch = new TreeMap<>(); - List>> firstMatches = new LinkedList<>(); - - Map caps = toMap(source.get("capabilities")); - if (caps != null) { - Map always = toMap(caps.get("alwaysMatch")); - if (always != null) { - alwaysMatch.putAll(always); - dialects.add(Dialect.W3C); - } - Object raw = caps.get("firstMatch"); - if (raw instanceof Collection) { - ((Collection) raw).stream() - .map(NewSessionPayload::toMap) - .filter(Objects::nonNull) - .forEach(m -> firstMatches.add(() -> m)); - dialects.add(Dialect.W3C); - } - if (firstMatches.isEmpty()) { - firstMatches.add(ImmutableMap::of); + Set usedKeys = new HashSet<>(); + + // Are there any values we care want to pull out into a mapping of their own? + List> firsts = adapters.stream() + .map(adapter -> adapter.apply(oss)) + .filter(Objects::nonNull) + .filter(map -> !map.isEmpty()) + .map(map -> + map.entrySet().stream() + .filter(entry -> entry.getKey() != null) + .filter(entry -> ACCEPTED_W3C_PATTERNS.test(entry.getKey())) + .filter(entry -> entry.getValue() != null) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))) + .peek(map -> usedKeys.addAll(map.keySet())) + .collect(ImmutableList.toImmutableList()); + if (firsts.isEmpty()) { + firsts = ImmutableList.of(ImmutableMap.of()); } - } - - byte[] json = new Json().toJson(source).getBytes(UTF_8); - - return new Sources( - () -> new ByteArrayInputStream(json), - json.length, - () -> oss, - () -> alwaysMatch, - firstMatches, - dialects); - } - - private Sources diskBackedSource(Reader source) throws IOException { - LOG.fine("Disk-based payload for " + source); - // Copy the original payload to disk - Path payload = root.resolve("original-payload.json"); - try (Writer out = Files.newBufferedWriter(payload, UTF_8)) { - CharStreams.copy(source, out); + // Are there any remaining unused keys? + Map always = oss.entrySet().stream() + .filter(entry -> !usedKeys.contains(entry.getKey())) + .filter(entry -> entry.getValue() != null) + .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); + + // Firsts contains at least one entry, always contains everything else. Let's combine them + // into the stream to form a unified set of capabilities. Woohoo! + fromOss = firsts.stream() + .map(first -> ImmutableMap.builder().putAll(always).putAll(first).build()) + .map(this::convertOssToW3C) + .map(map -> map.entrySet().stream() + .filter(entry -> ACCEPTED_W3C_PATTERNS.test(entry.getKey())) + .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue))) + .filter(map -> !map.isEmpty()) + .map(map -> (Map) map); + } else { + fromOss = Stream.of(); } - try ( - Reader in = Files.newBufferedReader(payload); - JsonInput jin = json.newInput(in)) { - Set dialects = new TreeSet<>(); - Supplier> oss = null; - Supplier> always = ImmutableMap::of; - List>> first = new LinkedList<>(); - - jin.beginObject(); - - while (jin.hasNext()) { - switch (jin.nextName()) { - case "capabilities": - jin.beginObject(); - while (jin.hasNext()) { - switch (jin.nextName()) { - - case "alwaysMatch": - Path a = write("always-match.json", jin); - always = () -> read(a); - dialects.add(Dialect.W3C); - break; - - case "firstMatch": - jin.beginArray(); - int i = 0; - while (jin.hasNext()) { - Path f = write("first-match-" + i + ".json", jin); - first.add(() -> read(f)); - i++; - } - jin.endArray(); - dialects.add(Dialect.W3C); - break; - - default: - jin.skipValue(); - } - } - jin.endObject(); - break; - - case "desiredCapabilities": - Path ossCaps = write("oss.json", jin); - oss = () -> read(ossCaps); - dialects.add(Dialect.OSS); - break; + Stream> fromW3c = null; + Map alwaysMatch = getAlwaysMatch(); + Collection> firsts = getFirstMatches(); - default: - jin.skipValue(); - } + if (alwaysMatch == null && firsts == null) { + fromW3c = Stream.of(); // No W3C capabilities. + } else { + if (alwaysMatch == null) { + alwaysMatch = ImmutableMap.of(); } - jin.endObject(); - - if (first.isEmpty()) { - first.add(ImmutableMap::of); + Map always = alwaysMatch; // Keep the comoiler happy. + if (firsts == null) { + firsts = ImmutableList.of(ImmutableMap.of()); } - return new Sources( - () -> { - try { - return Files.newInputStream(payload); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }, - Files.size(payload), - oss, - always, - first, - dialects); - } finally { - source.close(); - } - } - - private Path write(String fileName, JsonInput json) { - Path out = root.resolve(fileName); - - try (Writer writer = Files.newBufferedWriter(out, UTF_8, TRUNCATE_EXISTING, CREATE); - JsonOutput jsonOut = this.json.newOutput(writer)) { - jsonOut.write(json, MAP_TYPE); - } catch (IOException e) { - throw new UncheckedIOException(e); + fromW3c = firsts.stream() + .map(first -> ImmutableMap.builder().putAll(always).putAll(first).build()); } - return out; + return Stream.concat(fromOss, fromW3c); } - private Map read(Path path) { - try (Reader reader = Files.newBufferedReader(path, UTF_8); - JsonInput jin = json.newInput(reader)) { - return jin.read(MAP_TYPE); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } + private Map convertOssToW3C(ImmutableMap capabilities) { + Map toReturn = new TreeMap<>(); + toReturn.putAll(capabilities); - private static Map toMap(Object obj) { - if (!(obj instanceof Map)) { - return null; + // Platform name + if (capabilities.containsKey("platform") && !capabilities.containsKey("platformName")) { + toReturn.put("platformName", String.valueOf(capabilities.get("platform"))); } - return ((Map) obj).entrySet() - .stream() - .filter(e -> e.getKey() != null) - .filter(e -> e.getValue() != null) - .collect(ImmutableMap.toImmutableMap(e -> String.valueOf(e.getKey()), Map.Entry::getValue)); + return toReturn; } - /** - * Stream the {@link Capabilities} encoded in the payload used to create this instance. The - * {@link Stream} will start with a {@link Capabilities} object matching the OSS capabilities, and - * will then expand each of the "{@code firstMatch}" and "{@code alwaysMatch}" contents as defined - * in the W3C WebDriver spec. - *

- * The OSS {@link Capabilities} are listed first because converting the OSS capabilities to the - * equivalent W3C capabilities isn't particularly easy, so it's hoped that this approach gives us - * the most compatible implementation. - */ - public Stream stream() throws IOException { - Stream> mapStream; - - if (getDownstreamDialects().contains(Dialect.W3C)) { - Map always = sources.getAlwaysMatch().get(); - mapStream = sources.getFirstMatch().stream() - .map(Supplier::get) - .map(m -> ImmutableMap.builder().putAll(always).putAll(m).build()); - } else if (getDownstreamDialects().contains(Dialect.OSS)) { - mapStream = Stream.of(sources.getOss().get()); - } else { - mapStream = Stream.of(ImmutableMap.of()); + private Map getAlwaysMatch() throws IOException { + CharSource charSource = backingStore.asByteSource().asCharSource(UTF_8); + try (Reader reader = charSource.openBufferedStream(); + JsonInput input = json.newInput(reader)) { + input.beginObject(); + while (input.hasNext()) { + String name = input.nextName(); + if ("capabilities".equals(name)) { + input.beginObject(); + while (input.hasNext()) { + name = input.nextName(); + if ("alwaysMatch".equals(name)) { + return input.read(MAP_TYPE); + } else { + input.skipValue(); + } + } + input.endObject(); + } else { + input.skipValue(); + } + } } - - return mapStream.map(ImmutableCapabilities::new); + return null; } - public ImmutableSet getDownstreamDialects() { - return sources.getDialects().isEmpty() ? - ImmutableSet.of(DEFAULT_DIALECT) : - sources.getDialects(); - } - - public Supplier getPayload() { - return sources.getOriginalPayload(); - } - - public long getPayloadSize() { - return sources.getPayloadSize(); - } - - @Override - public void close() { - if (root != null) { - FileHandler.delete(root.toAbsolutePath().toFile()); + private Collection> getFirstMatches() throws IOException { + CharSource charSource = backingStore.asByteSource().asCharSource(UTF_8); + try (Reader reader = charSource.openBufferedStream(); + JsonInput input = json.newInput(reader)) { + input.beginObject(); + while (input.hasNext()) { + String name = input.nextName(); + if ("capabilities".equals(name)) { + input.beginObject(); + while (input.hasNext()) { + name = input.nextName(); + if ("firstMatch".equals(name)) { + return input.read(new TypeToken>>(){}.getType()); + } else { + input.skipValue(); + } + } + input.endObject(); + } else { + input.skipValue(); + } + } } + return null; } - private static class Sources { - - private final Supplier originalPayload; - private final long payloadSizeInBytes; - private final Supplier> oss; - private final Supplier> alwaysMatch; - private final List>> firstMatch; - private final ImmutableSet dialects; - - Sources( - Supplier originalPayload, - long payloadSizeInBytes, - Supplier> oss, - Supplier> alwaysMatch, - List>> firstMatch, - Set dialects) { - this.originalPayload = originalPayload; - this.payloadSizeInBytes = payloadSizeInBytes; - this.oss = oss; - this.alwaysMatch = alwaysMatch; - this.firstMatch = firstMatch; - this.dialects = ImmutableSet.copyOf(dialects); - } - - Supplier getOriginalPayload() { - return originalPayload; - } - - Supplier> getOss() { - return oss; - } + private Map applyTransforms(Map caps) { + Queue> toExamine = new LinkedList<>(); + toExamine.addAll(caps.entrySet()); + Set seenKeys = new HashSet<>(); + Map toReturn = new TreeMap<>(); - Supplier> getAlwaysMatch() { - return alwaysMatch; - } + // Take each entry and apply the transforms + while (!toExamine.isEmpty()) { + Map.Entry entry = toExamine.remove(); + seenKeys.add(entry.getKey()); - List>> getFirstMatch() { - return firstMatch; - } + if (entry.getValue() == null) { + continue; + } - ImmutableSet getDialects() { - return dialects; - } + for (CapabilityTransform transform : transforms) { + Collection> result = transform.apply(entry); + if (result == null) { + toReturn.remove(entry.getKey()); + break; + } - public long getPayloadSize() { - return payloadSizeInBytes; + for (Map.Entry newEntry : result) { + if (!seenKeys.contains(newEntry.getKey())) { + toExamine.add(newEntry); + } else { + if (newEntry.getKey().equals(entry.getKey())) { + entry = newEntry; + } + toReturn.put(newEntry.getKey(), newEntry.getValue()); + } + } + } } + return toReturn; } } \ No newline at end of file From 9a34f4176c4a8fd064d3dff953b188e958e75b4a Mon Sep 17 00:00:00 2001 From: shs96c Date: Fri, 9 Feb 2018 22:50:20 +0300 Subject: [PATCH 04/15] Rookie mistake: only include the w3c capabilities in the w3c capabilities --- .../java/org/openqa/selenium/remote/NewSessionPayload.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java index 3b6afdeff..86c88d163 100644 --- a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java +++ b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java @@ -239,10 +239,7 @@ public void writeTo(Appendable appendable) throws IOException { json.name("firstMatch"); json.beginArray(); //noinspection unchecked - stream() - .map(Capabilities::asMap) - .map(map -> (Map) map) - .forEach(map -> json.write(map, MAP_TYPE)); + getW3C().forEach(map -> json.write(map, MAP_TYPE)); json.endArray(); json.endObject(); // Close "capabilities" object From f860f33ae895d88ca059fdb3ecd671df6d384414 Mon Sep 17 00:00:00 2001 From: Sergey Tikhomirov Date: Fri, 9 Feb 2018 22:51:44 +0300 Subject: [PATCH 05/15] Fix tests failing because of ProtocolHandshake --- .../selenium/remote/NewSessionPayload.java | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java index 86c88d163..be10d4638 100644 --- a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java +++ b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java @@ -217,10 +217,13 @@ public void writeTo(Appendable appendable) throws IOException { try (JsonOutput json = new Json().newOutput(appendable)) { json.beginObject(); - @SuppressWarnings("unchecked") - Map first = (Map) stream().findFirst() - .orElse(new ImmutableCapabilities()) - .asMap(); + Map first = getOss(); + if (first == null) { + //noinspection unchecked + first = (Map) stream().findFirst() + .orElse(new ImmutableCapabilities()) + .asMap(); + } // Write the first capability we get as the desired capability. json.name("desiredCapabilities"); @@ -295,7 +298,7 @@ public Stream stream() throws IOException { } public ImmutableSet getDownstreamDialects() { - return dialects.isEmpty() ? ImmutableSet.of(Dialect.OSS) : dialects; + return dialects.isEmpty() ? ImmutableSet.of(DEFAULT_DIALECT) : dialects; } @Override @@ -325,7 +328,7 @@ private Stream> getW3C() throws IOException { // then add magic to generate each of the w3c capabilities. For the sake of simplicity, we're // going to make the (probably wrong) assumption we can hold all of the firstMatch values and // alwaysMatch value in memory at the same time. - Map oss = getOss(); + Map oss = convertOssToW3C(getOss()); Stream> fromOss; if (oss != null) { Set usedKeys = new HashSet<>(); @@ -357,11 +360,10 @@ private Stream> getW3C() throws IOException { // into the stream to form a unified set of capabilities. Woohoo! fromOss = firsts.stream() .map(first -> ImmutableMap.builder().putAll(always).putAll(first).build()) - .map(this::convertOssToW3C) + .map(this::applyTransforms) .map(map -> map.entrySet().stream() .filter(entry -> ACCEPTED_W3C_PATTERNS.test(entry.getKey())) .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue))) - .filter(map -> !map.isEmpty()) .map(map -> (Map) map); } else { fromOss = Stream.of(); @@ -386,10 +388,14 @@ private Stream> getW3C() throws IOException { .map(first -> ImmutableMap.builder().putAll(always).putAll(first).build()); } - return Stream.concat(fromOss, fromW3c); + return Stream.concat(fromOss, fromW3c).distinct(); } - private Map convertOssToW3C(ImmutableMap capabilities) { + private Map convertOssToW3C(Map capabilities) { + if (capabilities == null) { + return null; + } + Map toReturn = new TreeMap<>(); toReturn.putAll(capabilities); From ed5fba8fb8520d3ee38666f88606d6d99df265f0 Mon Sep 17 00:00:00 2001 From: shs96c Date: Fri, 9 Feb 2018 22:53:26 +0300 Subject: [PATCH 06/15] Ensure that we forward metadata in the new session payload --- .../selenium/remote/NewSessionPayload.java | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java index be10d4638..4bbe91cc3 100644 --- a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java +++ b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java @@ -219,8 +219,7 @@ public void writeTo(Appendable appendable) throws IOException { Map first = getOss(); if (first == null) { - //noinspection unchecked - first = (Map) stream().findFirst() + first = stream().findFirst() .orElse(new ImmutableCapabilities()) .asMap(); } @@ -246,10 +245,36 @@ public void writeTo(Appendable appendable) throws IOException { json.endArray(); json.endObject(); // Close "capabilities" object + + writeMetaData(json); + json.endObject(); } } + private void writeMetaData(JsonOutput out) throws IOException { + CharSource charSource = backingStore.asByteSource().asCharSource(UTF_8); + try (Reader reader = charSource.openBufferedStream(); + JsonInput input = json.newInput(reader)) { + input.beginObject(); + while (input.hasNext()) { + String name = input.nextName(); + switch (name) { + case "capabilities": + case "desiredCapabilities": + case "requiredCapabilities": + input.skipValue(); + break; + + default: + out.name(name); + out.write(input.read(Object.class), Object.class); + break; + } + } + } + } + private void streamW3CProtocolParameters(JsonOutput out, Map des) { // Technically we should be building up a combination of "alwaysMatch" and "firstMatch" options. // We're going to do a little processing to figure out what we might be able to do, and assume From 67588ecacc07575c9f4d726b8cba4be6c5486bc6 Mon Sep 17 00:00:00 2001 From: Alexei Barantsev Date: Fri, 9 Feb 2018 23:08:09 +0300 Subject: [PATCH 07/15] Introducing new string constant for "platformName" --- .../java/org/openqa/selenium/remote/NewSessionPayload.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java index 4bbe91cc3..d263d069d 100644 --- a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java +++ b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java @@ -19,6 +19,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.openqa.selenium.json.Json.MAP_TYPE; +import static org.openqa.selenium.remote.CapabilityType.PLATFORM; +import static org.openqa.selenium.remote.CapabilityType.PLATFORM_NAME; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -425,8 +427,8 @@ private Map convertOssToW3C(Map capabilities) { toReturn.putAll(capabilities); // Platform name - if (capabilities.containsKey("platform") && !capabilities.containsKey("platformName")) { - toReturn.put("platformName", String.valueOf(capabilities.get("platform"))); + if (capabilities.containsKey(PLATFORM) && !capabilities.containsKey(PLATFORM_NAME)) { + toReturn.put(PLATFORM_NAME, String.valueOf(capabilities.get(PLATFORM))); } return toReturn; From 264476adb725394ac576a8356eb7464dcd66c08c Mon Sep 17 00:00:00 2001 From: Sergey Tikhomirov Date: Fri, 9 Feb 2018 23:10:48 +0300 Subject: [PATCH 08/15] Bump the java libraries we depend on to their latest versions Using the magic of Buck's "maven-importer" and the following maven coordinates: ``` 'org.seleniumhq.selenium:htmlunit-driver:jar:2.28.5' \ 'junit:junit:jar:4.12' \ 'net.bytebuddy:byte-buddy:jar:1.7.9' \ 'com.google.code.gson:gson:jar:2.8.2' \ 'com.google.guava:guava:jar:23.6-jre' \ 'org.apache.commons:commons-exec:jar:1.3' \ 'org.eclipse.jetty:jetty-security:jar:9.4.8.v20171121' \ 'org.testng:testng:jar:6.13.1' \ 'org.pantsbuild:jarjar:jar:1.6.5' \ 'org.eclipse.jetty:jetty-util:jar:9.4.8.v20171121' \ 'org.eclipse.jetty:jetty-server:jar:9.4.8.v20171121' \ 'org.eclipse.jetty:jetty-servlet:jar:9.4.8.v20171121' \ 'org.hamcrest:hamcrest-library:jar:1.3' \ 'com.github.javaparser:javaparser-core:jar:3.5.7' \ 'org.eclipse.jetty:jetty-jmx:jar:9.4.8.v20171121' \ 'net.jcip:jcip-annotations:jar:1.0' \ 'org.yaml:snakeyaml:jar:1.19' \ 'org.mockito:mockito-core:jar:2.13.0' \ 'io.netty:netty-all:jar:4.1.19.Final' \ 'org.eclipse.mylyn.github:org.eclipse.egit.github.core:jar:2.1.5' \ 'org.littleshoot:littleproxy:jar:1.1.2' \ 'org.slf4j:slf4j-jdk14:jar:1.7.25' ``` The version of LittleProxy we use is a snapshot and is the old version from previously with dependencies updated. --- .../java/org/openqa/selenium/remote/NewSessionPayload.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java index d263d069d..90b5918fd 100644 --- a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java +++ b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java @@ -18,6 +18,7 @@ package org.openqa.selenium.remote; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.openqa.selenium.json.Json.LIST_OF_MAPS_TYPE; import static org.openqa.selenium.json.Json.MAP_TYPE; import static org.openqa.selenium.remote.CapabilityType.PLATFORM; import static org.openqa.selenium.remote.CapabilityType.PLATFORM_NAME; @@ -31,7 +32,6 @@ import com.google.common.io.CharSource; import com.google.common.io.CharStreams; import com.google.common.io.FileBackedOutputStream; -import com.google.gson.reflect.TypeToken; import org.openqa.selenium.Capabilities; import org.openqa.selenium.ImmutableCapabilities; @@ -472,7 +472,7 @@ private Collection> getFirstMatches() throws IOException { while (input.hasNext()) { name = input.nextName(); if ("firstMatch".equals(name)) { - return input.read(new TypeToken>>(){}.getType()); + return input.read(LIST_OF_MAPS_TYPE); } else { input.skipValue(); } From fbb6e58eb5d299b02a79ee111ecf0f43ae18b4ca Mon Sep 17 00:00:00 2001 From: Sergey Tikhomirov Date: Fri, 9 Feb 2018 23:12:24 +0300 Subject: [PATCH 09/15] Use `Capabilities` rather than `ImmutableCapabilities` in mutators --- .../java/org/openqa/selenium/remote/NewSessionPayload.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java index 90b5918fd..98458fccb 100644 --- a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java +++ b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java @@ -221,7 +221,8 @@ public void writeTo(Appendable appendable) throws IOException { Map first = getOss(); if (first == null) { - first = stream().findFirst() + //noinspection unchecked + first = (Map) stream().findFirst() .orElse(new ImmutableCapabilities()) .asMap(); } @@ -309,7 +310,7 @@ private void streamW3CProtocolParameters(JsonOutput out, Map des * equivalent W3C capabilities isn't particularly easy, so it's hoped that this approach gives us * the most compatible implementation. */ - public Stream stream() throws IOException { + public Stream stream() throws IOException { // OSS first Stream> oss = Stream.of(getOss()); From 18552913bdd6b9025d51e4e4a1b74509d80b0797 Mon Sep 17 00:00:00 2001 From: Sergey Tikhomirov Date: Sun, 11 Feb 2018 03:01:52 +0300 Subject: [PATCH 10/15] Update to selenium 3.9.1. Changes of NewSessionPayload. --- build.gradle | 2 +- .../remote/MobileCapabilityType.java | 11 +-- .../selenium/remote/NewSessionPayload.java | 90 +++++++++++++++---- 3 files changed, 79 insertions(+), 24 deletions(-) diff --git a/build.gradle b/build.gradle index d89cc88e8..baca24640 100644 --- a/build.gradle +++ b/build.gradle @@ -50,7 +50,7 @@ compileJava { ] } -ext.seleniumVersion = '3.8.1' +ext.seleniumVersion = '3.9.1' dependencies { compile ("org.seleniumhq.selenium:selenium-java:${seleniumVersion}") { diff --git a/src/main/java/io/appium/java_client/remote/MobileCapabilityType.java b/src/main/java/io/appium/java_client/remote/MobileCapabilityType.java index 40077436f..7a1b1fa01 100644 --- a/src/main/java/io/appium/java_client/remote/MobileCapabilityType.java +++ b/src/main/java/io/appium/java_client/remote/MobileCapabilityType.java @@ -31,11 +31,6 @@ public interface MobileCapabilityType extends CapabilityType { */ String AUTOMATION_NAME = "automationName"; - /** - * Which mobile OS platform to use. - */ - String PLATFORM_NAME = "platformName"; - /** * Mobile OS version. */ @@ -120,4 +115,10 @@ public interface MobileCapabilityType extends CapabilityType { * (e.g., the start and end of each command, etc.). Defaults to false. */ String EVENT_TIMINGS = "eventTimings"; + + /** + * This is the flag which forces server to switch to the mobile WSONWP. + * If {@code false} when it is switched to W3C mode. + */ + String FORCE_MJSONWP = "forceMjsonwp"; } diff --git a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java index 98458fccb..ff5838f8c 100644 --- a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java +++ b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java @@ -17,7 +17,11 @@ package org.openqa.selenium.remote; +import static io.appium.java_client.remote.MobileCapabilityType.FORCE_MJSONWP; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Arrays.asList; +import static java.util.Optional.ofNullable; +import static java.util.stream.Collectors.toList; import static org.openqa.selenium.json.Json.LIST_OF_MAPS_TYPE; import static org.openqa.selenium.json.Json.MAP_TYPE; import static org.openqa.selenium.remote.CapabilityType.PLATFORM; @@ -33,6 +37,10 @@ import com.google.common.io.CharStreams; import com.google.common.io.FileBackedOutputStream; +import io.appium.java_client.remote.AndroidMobileCapabilityType; +import io.appium.java_client.remote.IOSMobileCapabilityType; +import io.appium.java_client.remote.MobileCapabilityType; +import io.appium.java_client.remote.YouiEngineCapabilityType; import org.openqa.selenium.Capabilities; import org.openqa.selenium.ImmutableCapabilities; import org.openqa.selenium.json.Json; @@ -56,17 +64,8 @@ import java.io.Reader; import java.io.StringReader; import java.io.Writer; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Queue; -import java.util.ServiceLoader; -import java.util.Set; -import java.util.TreeMap; +import java.util.*; +import java.util.function.Function; import java.util.function.Predicate; import java.util.logging.Logger; import java.util.regex.Pattern; @@ -77,9 +76,15 @@ public class NewSessionPayload implements Closeable { private static final Logger LOG = Logger.getLogger(NewSessionPayload.class.getName()); + private static final List APPIUM_CAPABILITIES = ImmutableList.builder() + .addAll(getAppiumCapabilities(MobileCapabilityType.class)) + .addAll(getAppiumCapabilities(AndroidMobileCapabilityType.class)) + .addAll(getAppiumCapabilities(IOSMobileCapabilityType.class)) + .addAll(getAppiumCapabilities(YouiEngineCapabilityType.class)).build(); private final Set adapters; private final Set transforms; + private final boolean forceMobileJSONWP; private static final Dialect DEFAULT_DIALECT = Dialect.OSS; private final static Predicate ACCEPTED_W3C_PATTERNS = Stream.of( @@ -101,24 +106,45 @@ public class NewSessionPayload implements Closeable { private final FileBackedOutputStream backingStore; private final ImmutableSet dialects; + private static List getAppiumCapabilities(Class capabilityList) { + return Arrays.stream(capabilityList.getDeclaredFields()).map(field -> { + field.setAccessible(true); + try { + return field.get(capabilityList).toString(); + } catch (IllegalAccessException e) { + throw new IllegalArgumentException(e); + } + }).filter(s -> !FORCE_MJSONWP.equals(s)).collect(toList()); + } + public static NewSessionPayload create(Capabilities caps) throws IOException { // We need to convert the capabilities into a new session payload. At this point we're dealing // with references, so I'm Just Sure This Will Be Fine. - return create(ImmutableMap.of("desiredCapabilities", caps.asMap())); + boolean forceMobileJSONWP = ofNullable(caps.getCapability(FORCE_MJSONWP)) + .map(o -> { + if (Boolean.class.isAssignableFrom(o.getClass())) { + return Boolean.class.cast(o); + } + return false; + }).orElse(false); + HashMap capabilityMap = new HashMap<>(caps.asMap()); + capabilityMap.remove(FORCE_MJSONWP); + return create(ImmutableMap.of("desiredCapabilities", capabilityMap), forceMobileJSONWP); } - public static NewSessionPayload create(Map source) throws IOException { + public static NewSessionPayload create(Map source, boolean forceMobileJSONWP) throws IOException { Objects.requireNonNull(source, "Payload must be set"); String json = new Json().toJson(source); - return new NewSessionPayload(new StringReader(json)); + return new NewSessionPayload(new StringReader(json), forceMobileJSONWP); } - public static NewSessionPayload create(Reader source) throws IOException { - return new NewSessionPayload(source); + public static NewSessionPayload create(Reader source, boolean forceMobileJSONWP) throws IOException { + return new NewSessionPayload(source, forceMobileJSONWP); } - private NewSessionPayload(Reader source) throws IOException { + private NewSessionPayload(Reader source, boolean forceMobileJSONWP) throws IOException { + this.forceMobileJSONWP = forceMobileJSONWP; // Dedicate up to 10% of all RAM or 20% of available RAM (whichever is smaller) to storing this // payload. int threshold = (int) Math.min( @@ -358,6 +384,7 @@ private Stream> getW3C() throws IOException { // alwaysMatch value in memory at the same time. Map oss = convertOssToW3C(getOss()); Stream> fromOss; + if (oss != null) { Set usedKeys = new HashSet<>(); @@ -390,7 +417,34 @@ private Stream> getW3C() throws IOException { .map(first -> ImmutableMap.builder().putAll(always).putAll(first).build()) .map(this::applyTransforms) .map(map -> map.entrySet().stream() - .filter(entry -> ACCEPTED_W3C_PATTERNS.test(entry.getKey())) + .filter(entry -> { + if (forceMobileJSONWP) { + return ACCEPTED_W3C_PATTERNS.test(entry.getKey()); + } + else { + return true; + } + }) + .map((Function, Map.Entry>) stringObjectEntry -> + new Map.Entry() { + @Override + public String getKey() { + String key = stringObjectEntry.getKey(); + if (APPIUM_CAPABILITIES.contains(key) && !forceMobileJSONWP) { + return "appium:" + key; + } + return key; + } + + @Override + public Object getValue() { + return stringObjectEntry.getValue(); + } + + @Override + public Object setValue(Object value) { + return stringObjectEntry.setValue(value); + }}) .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue))) .map(map -> (Map) map); } else { From a79c6248ca520092d4318141fcd37b38fdbdb376 Mon Sep 17 00:00:00 2001 From: Sergey Tikhomirov Date: Mon, 12 Feb 2018 00:55:21 +0300 Subject: [PATCH 11/15] fixing out: - removal of redundant code from the original NewSessionPayload --- .../remote/MobileCapabilityType.java | 2 +- .../selenium/remote/NewSessionPayload.java | 60 ++----------------- 2 files changed, 7 insertions(+), 55 deletions(-) diff --git a/src/main/java/io/appium/java_client/remote/MobileCapabilityType.java b/src/main/java/io/appium/java_client/remote/MobileCapabilityType.java index 7a1b1fa01..ef548978f 100644 --- a/src/main/java/io/appium/java_client/remote/MobileCapabilityType.java +++ b/src/main/java/io/appium/java_client/remote/MobileCapabilityType.java @@ -118,7 +118,7 @@ public interface MobileCapabilityType extends CapabilityType { /** * This is the flag which forces server to switch to the mobile WSONWP. - * If {@code false} when it is switched to W3C mode. + * If {@code false} then it is switched to W3C mode. */ String FORCE_MJSONWP = "forceMjsonwp"; } diff --git a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java index ff5838f8c..2aaf65190 100644 --- a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java +++ b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java @@ -13,13 +13,13 @@ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations -// under the License. +// under the License.writeTo package org.openqa.selenium.remote; +import static com.google.common.collect.ImmutableMap.of; import static io.appium.java_client.remote.MobileCapabilityType.FORCE_MJSONWP; import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Arrays.asList; import static java.util.Optional.ofNullable; import static java.util.stream.Collectors.toList; import static org.openqa.selenium.json.Json.LIST_OF_MAPS_TYPE; @@ -67,7 +67,6 @@ import java.util.*; import java.util.function.Function; import java.util.function.Predicate; -import java.util.logging.Logger; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -75,7 +74,6 @@ public class NewSessionPayload implements Closeable { - private static final Logger LOG = Logger.getLogger(NewSessionPayload.class.getName()); private static final List APPIUM_CAPABILITIES = ImmutableList.builder() .addAll(getAppiumCapabilities(MobileCapabilityType.class)) .addAll(getAppiumCapabilities(AndroidMobileCapabilityType.class)) @@ -86,7 +84,6 @@ public class NewSessionPayload implements Closeable { private final Set transforms; private final boolean forceMobileJSONWP; - private static final Dialect DEFAULT_DIALECT = Dialect.OSS; private final static Predicate ACCEPTED_W3C_PATTERNS = Stream.of( "^[\\w-]+:.*$", "^acceptInsecureCerts$", @@ -104,7 +101,6 @@ public class NewSessionPayload implements Closeable { private final Json json = new Json(); private final FileBackedOutputStream backingStore; - private final ImmutableSet dialects; private static List getAppiumCapabilities(Class capabilityList) { return Arrays.stream(capabilityList.getDeclaredFields()).map(field -> { @@ -118,8 +114,6 @@ private static List getAppiumCapabilities(Class capabilityList) { } public static NewSessionPayload create(Capabilities caps) throws IOException { - // We need to convert the capabilities into a new session payload. At this point we're dealing - // with references, so I'm Just Sure This Will Be Fine. boolean forceMobileJSONWP = ofNullable(caps.getCapability(FORCE_MJSONWP)) .map(o -> { if (Boolean.class.isAssignableFrom(o.getClass())) { @@ -127,22 +121,14 @@ public static NewSessionPayload create(Capabilities caps) throws IOException { } return false; }).orElse(false); + HashMap capabilityMap = new HashMap<>(caps.asMap()); capabilityMap.remove(FORCE_MJSONWP); - return create(ImmutableMap.of("desiredCapabilities", capabilityMap), forceMobileJSONWP); - } - - public static NewSessionPayload create(Map source, boolean forceMobileJSONWP) throws IOException { - Objects.requireNonNull(source, "Payload must be set"); - + Map source = of("desiredCapabilities", capabilityMap); String json = new Json().toJson(source); return new NewSessionPayload(new StringReader(json), forceMobileJSONWP); } - public static NewSessionPayload create(Reader source, boolean forceMobileJSONWP) throws IOException { - return new NewSessionPayload(source, forceMobileJSONWP); - } - private NewSessionPayload(Reader source, boolean forceMobileJSONWP) throws IOException { this.forceMobileJSONWP = forceMobileJSONWP; // Dedicate up to 10% of all RAM or 20% of available RAM (whichever is smaller) to storing this @@ -184,7 +170,6 @@ private NewSessionPayload(Reader source, boolean forceMobileJSONWP) throws IOExc if (getAlwaysMatch() != null || getFirstMatches() != null) { dialects.add(Dialect.W3C); } - this.dialects = dialects.build(); validate(); } @@ -304,28 +289,6 @@ private void writeMetaData(JsonOutput out) throws IOException { } } - private void streamW3CProtocolParameters(JsonOutput out, Map des) { - // Technically we should be building up a combination of "alwaysMatch" and "firstMatch" options. - // We're going to do a little processing to figure out what we might be able to do, and assume - // that people don't really understand the difference between required and desired (which is - // commonly the case). Wish us luck. Looking at the current implementations, people may have - // set options for multiple browsers, in which case a compliant W3C remote end won't start - // a session. If we find this, then we create multiple firstMatch capabilities. Furrfu. - // The table of options are: - // - // Chrome: chromeOptions - // Firefox: moz:.*, firefox_binary, firefox_profile, marionette - // Edge: none given - // IEDriver: ignoreZoomSetting, initialBrowserUrl, enableElementCacheCleanup, - // browserAttachTimeout, enablePersistentHover, requireWindowFocus, logFile, logLevel, host, - // extractPath, silent, ie.* - // Opera: operaOptions - // SafariDriver: safari.options - // - // We can't use the constants defined in the classes because it would introduce circular - // dependencies between the remote library and the implementations. Yay! - } - /** * Stream the {@link Capabilities} encoded in the payload used to create this instance. The * {@link Stream} will start with a {@link Capabilities} object matching the OSS capabilities, and @@ -351,10 +314,6 @@ public Stream stream() throws IOException { .map(ImmutableCapabilities::new); } - public ImmutableSet getDownstreamDialects() { - return dialects.isEmpty() ? ImmutableSet.of(DEFAULT_DIALECT) : dialects; - } - @Override public void close() throws IOException { backingStore.reset(); @@ -417,14 +376,7 @@ private Stream> getW3C() throws IOException { .map(first -> ImmutableMap.builder().putAll(always).putAll(first).build()) .map(this::applyTransforms) .map(map -> map.entrySet().stream() - .filter(entry -> { - if (forceMobileJSONWP) { - return ACCEPTED_W3C_PATTERNS.test(entry.getKey()); - } - else { - return true; - } - }) + .filter(entry -> !forceMobileJSONWP || ACCEPTED_W3C_PATTERNS.test(entry.getKey())) .map((Function, Map.Entry>) stringObjectEntry -> new Map.Entry() { @Override @@ -451,7 +403,7 @@ public Object setValue(Object value) { fromOss = Stream.of(); } - Stream> fromW3c = null; + Stream> fromW3c; Map alwaysMatch = getAlwaysMatch(); Collection> firsts = getFirstMatches(); From c23e05b9b7ce4059545495b5cd69920606883b3c Mon Sep 17 00:00:00 2001 From: Sergey Tikhomirov Date: Mon, 12 Feb 2018 01:13:57 +0300 Subject: [PATCH 12/15] Removal of "magic" strings --- .../selenium/remote/NewSessionPayload.java | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java index 2aaf65190..4bc498c6c 100644 --- a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java +++ b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java @@ -79,6 +79,12 @@ public class NewSessionPayload implements Closeable { .addAll(getAppiumCapabilities(AndroidMobileCapabilityType.class)) .addAll(getAppiumCapabilities(IOSMobileCapabilityType.class)) .addAll(getAppiumCapabilities(YouiEngineCapabilityType.class)).build(); + private static final String APPIUM_PREFIX = "appium:"; + private static final String DESIRED_CAPABILITIES = "desiredCapabilities"; + private static final String CAPABILITIES = "capabilities"; + private static final String REQUIRED_CAPABILITIES = "requiredCapabilities"; + private static final String FIRST_MATCH = "firstMatch"; + private static final String ALWAYS_MATCH = "alwaysMatch"; private final Set adapters; private final Set transforms; @@ -124,7 +130,7 @@ public static NewSessionPayload create(Capabilities caps) throws IOException { HashMap capabilityMap = new HashMap<>(caps.asMap()); capabilityMap.remove(FORCE_MJSONWP); - Map source = of("desiredCapabilities", capabilityMap); + Map source = of(DESIRED_CAPABILITIES, capabilityMap); String json = new Json().toJson(source); return new NewSessionPayload(new StringReader(json), forceMobileJSONWP); } @@ -239,20 +245,20 @@ public void writeTo(Appendable appendable) throws IOException { } // Write the first capability we get as the desired capability. - json.name("desiredCapabilities"); + json.name(DESIRED_CAPABILITIES); json.write(first, MAP_TYPE); // And write the first capability for gecko13 - json.name("capabilities"); + json.name(CAPABILITIES); json.beginObject(); - json.name("desiredCapabilities"); + json.name(DESIRED_CAPABILITIES); json.write(first, MAP_TYPE); // Then write everything into the w3c payload. Because of the way we do this, it's easiest // to just populate the "firstMatch" section. The spec says it's fine to omit the // "alwaysMatch" field, so we do this. - json.name("firstMatch"); + json.name(FIRST_MATCH); json.beginArray(); //noinspection unchecked getW3C().forEach(map -> json.write(map, MAP_TYPE)); @@ -274,9 +280,9 @@ private void writeMetaData(JsonOutput out) throws IOException { while (input.hasNext()) { String name = input.nextName(); switch (name) { - case "capabilities": - case "desiredCapabilities": - case "requiredCapabilities": + case CAPABILITIES: + case DESIRED_CAPABILITIES: + case REQUIRED_CAPABILITIES: input.skipValue(); break; @@ -326,7 +332,7 @@ private Map getOss() throws IOException { input.beginObject(); while (input.hasNext()) { String name = input.nextName(); - if ("desiredCapabilities".equals(name)) { + if (DESIRED_CAPABILITIES.equals(name)) { return input.read(MAP_TYPE); } else { input.skipValue(); @@ -383,7 +389,7 @@ private Stream> getW3C() throws IOException { public String getKey() { String key = stringObjectEntry.getKey(); if (APPIUM_CAPABILITIES.contains(key) && !forceMobileJSONWP) { - return "appium:" + key; + return APPIUM_PREFIX + key; } return key; } @@ -448,11 +454,11 @@ private Map getAlwaysMatch() throws IOException { input.beginObject(); while (input.hasNext()) { String name = input.nextName(); - if ("capabilities".equals(name)) { + if (CAPABILITIES.equals(name)) { input.beginObject(); while (input.hasNext()) { name = input.nextName(); - if ("alwaysMatch".equals(name)) { + if (ALWAYS_MATCH.equals(name)) { return input.read(MAP_TYPE); } else { input.skipValue(); @@ -474,11 +480,11 @@ private Collection> getFirstMatches() throws IOException { input.beginObject(); while (input.hasNext()) { String name = input.nextName(); - if ("capabilities".equals(name)) { + if (CAPABILITIES.equals(name)) { input.beginObject(); while (input.hasNext()) { name = input.nextName(); - if ("firstMatch".equals(name)) { + if (FIRST_MATCH.equals(name)) { return input.read(LIST_OF_MAPS_TYPE); } else { input.skipValue(); From b694856ce3ab28a3dcf76923695b99606fe05783 Mon Sep 17 00:00:00 2001 From: Sergey Tikhomirov Date: Tue, 13 Feb 2018 01:52:59 +0300 Subject: [PATCH 13/15] Improving set of not so major issues --- .../selenium/remote/NewSessionPayload.java | 43 +++++++++---------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java index 4bc498c6c..53e85e4e5 100644 --- a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java +++ b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java @@ -18,6 +18,7 @@ package org.openqa.selenium.remote; import static com.google.common.collect.ImmutableMap.of; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static io.appium.java_client.remote.MobileCapabilityType.FORCE_MJSONWP; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Optional.ofNullable; @@ -58,6 +59,7 @@ import org.openqa.selenium.remote.session.StripAnyPlatform; import org.openqa.selenium.remote.session.W3CPlatformNameNormaliser; +import javax.annotation.Nullable; import java.io.Closeable; import java.io.IOException; import java.io.OutputStreamWriter; @@ -120,13 +122,10 @@ private static List getAppiumCapabilities(Class capabilityList) { } public static NewSessionPayload create(Capabilities caps) throws IOException { - boolean forceMobileJSONWP = ofNullable(caps.getCapability(FORCE_MJSONWP)) - .map(o -> { - if (Boolean.class.isAssignableFrom(o.getClass())) { - return Boolean.class.cast(o); - } - return false; - }).orElse(false); + boolean forceMobileJSONWP = + ofNullable(caps.getCapability(FORCE_MJSONWP)) + .map(o -> Boolean.class.isAssignableFrom(o.getClass()) && Boolean.class.cast(o)) + .orElse(false); HashMap capabilityMap = new HashMap<>(caps.asMap()); capabilityMap.remove(FORCE_MJSONWP); @@ -183,12 +182,12 @@ private NewSessionPayload(Reader source, boolean forceMobileJSONWP) throws IOExc private void validate() throws IOException { Map alwaysMatch = getAlwaysMatch(); if (alwaysMatch == null) { - alwaysMatch = ImmutableMap.of(); + alwaysMatch = of(); } Map always = alwaysMatch; Collection> firsts = getFirstMatches(); if (firsts == null) { - firsts = ImmutableList.of(ImmutableMap.of()); + firsts = ImmutableList.of(of()); } if (firsts.isEmpty()) { @@ -228,8 +227,7 @@ private void validate() throws IOException { throw new IllegalArgumentException( "Illegal key values seen in w3c capabilities: " + illegalKeys); } - }) - .forEach(map -> {}); + }); } public void writeTo(Appendable appendable) throws IOException { @@ -325,7 +323,7 @@ public void close() throws IOException { backingStore.reset(); } - private Map getOss() throws IOException { + private @Nullable Map getOss() throws IOException { CharSource charSource = backingStore.asByteSource().asCharSource(UTF_8); try (Reader reader = charSource.openBufferedStream(); JsonInput input = json.newInput(reader)) { @@ -334,9 +332,8 @@ private Map getOss() throws IOException { String name = input.nextName(); if (DESIRED_CAPABILITIES.equals(name)) { return input.read(MAP_TYPE); - } else { - input.skipValue(); } + input.skipValue(); } } return null; @@ -367,14 +364,14 @@ private Stream> getW3C() throws IOException { .peek(map -> usedKeys.addAll(map.keySet())) .collect(ImmutableList.toImmutableList()); if (firsts.isEmpty()) { - firsts = ImmutableList.of(ImmutableMap.of()); + firsts = ImmutableList.of(of()); } // Are there any remaining unused keys? Map always = oss.entrySet().stream() .filter(entry -> !usedKeys.contains(entry.getKey())) .filter(entry -> entry.getValue() != null) - .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); + .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); // Firsts contains at least one entry, always contains everything else. Let's combine them // into the stream to form a unified set of capabilities. Woohoo! @@ -447,7 +444,7 @@ private Map convertOssToW3C(Map capabilities) { return toReturn; } - private Map getAlwaysMatch() throws IOException { + private @Nullable Map getAlwaysMatch() throws IOException { CharSource charSource = backingStore.asByteSource().asCharSource(UTF_8); try (Reader reader = charSource.openBufferedStream(); JsonInput input = json.newInput(reader)) { @@ -473,7 +470,7 @@ private Map getAlwaysMatch() throws IOException { return null; } - private Collection> getFirstMatches() throws IOException { + private @Nullable Collection> getFirstMatches() throws IOException { CharSource charSource = backingStore.asByteSource().asCharSource(UTF_8); try (Reader reader = charSource.openBufferedStream(); JsonInput input = json.newInput(reader)) { @@ -524,12 +521,12 @@ private Map applyTransforms(Map caps) { for (Map.Entry newEntry : result) { if (!seenKeys.contains(newEntry.getKey())) { toExamine.add(newEntry); - } else { - if (newEntry.getKey().equals(entry.getKey())) { - entry = newEntry; - } - toReturn.put(newEntry.getKey(), newEntry.getValue()); + continue; + } + if (newEntry.getKey().equals(entry.getKey())) { + entry = newEntry; } + toReturn.put(newEntry.getKey(), newEntry.getValue()); } } } From afa1039db2a4dc1c7eabe627c23def0d56386d53 Mon Sep 17 00:00:00 2001 From: Sergey Tikhomirov Date: Tue, 13 Feb 2018 01:57:04 +0300 Subject: [PATCH 14/15] `convertOssToW3C` marked by `Nullable` --- src/main/java/org/openqa/selenium/remote/NewSessionPayload.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java index 53e85e4e5..8476dc48e 100644 --- a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java +++ b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java @@ -428,7 +428,7 @@ public Object setValue(Object value) { return Stream.concat(fromOss, fromW3c).distinct(); } - private Map convertOssToW3C(Map capabilities) { + private @Nullable Map convertOssToW3C(Map capabilities) { if (capabilities == null) { return null; } From 6ffd2ea625e315b4eb34c3a88c32fa481afc1c54 Mon Sep 17 00:00:00 2001 From: Sergey Tikhomirov Date: Tue, 13 Feb 2018 23:58:55 +0300 Subject: [PATCH 15/15] Fixing issuies before the merging --- .../remote/MobileCapabilityType.java | 6 ------ .../selenium/remote/NewSessionPayload.java | 20 +++++++++---------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/src/main/java/io/appium/java_client/remote/MobileCapabilityType.java b/src/main/java/io/appium/java_client/remote/MobileCapabilityType.java index ef548978f..46a873c23 100644 --- a/src/main/java/io/appium/java_client/remote/MobileCapabilityType.java +++ b/src/main/java/io/appium/java_client/remote/MobileCapabilityType.java @@ -56,12 +56,6 @@ public interface MobileCapabilityType extends CapabilityType { */ String APP = "app"; - /** - * Name of mobile web browser to automate. - * Should be an empty string if automating an app instead. - */ - String BROWSER_NAME = "browserName"; - /** * Unique device identifier of the connected physical device. */ diff --git a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java index 8476dc48e..02b238c5f 100644 --- a/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java +++ b/src/main/java/org/openqa/selenium/remote/NewSessionPayload.java @@ -172,7 +172,7 @@ private NewSessionPayload(Reader source, boolean forceMobileJSONWP) throws IOExc if (getOss() != null) { dialects.add(Dialect.OSS); } - if (getAlwaysMatch() != null || getFirstMatches() != null) { + if (getAlwaysMatch() != null || getFirstMatch() != null) { dialects.add(Dialect.W3C); } @@ -185,7 +185,7 @@ private void validate() throws IOException { alwaysMatch = of(); } Map always = alwaysMatch; - Collection> firsts = getFirstMatches(); + Collection> firsts = getFirstMatch(); if (firsts == null) { firsts = ImmutableList.of(of()); } @@ -400,7 +400,7 @@ public Object getValue() { public Object setValue(Object value) { return stringObjectEntry.setValue(value); }}) - .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue))) + .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue))) .map(map -> (Map) map); } else { fromOss = Stream.of(); @@ -408,17 +408,17 @@ public Object setValue(Object value) { Stream> fromW3c; Map alwaysMatch = getAlwaysMatch(); - Collection> firsts = getFirstMatches(); + Collection> firsts = getFirstMatch(); if (alwaysMatch == null && firsts == null) { fromW3c = Stream.of(); // No W3C capabilities. } else { if (alwaysMatch == null) { - alwaysMatch = ImmutableMap.of(); + alwaysMatch = of(); } Map always = alwaysMatch; // Keep the comoiler happy. if (firsts == null) { - firsts = ImmutableList.of(ImmutableMap.of()); + firsts = ImmutableList.of(of()); } fromW3c = firsts.stream() @@ -457,9 +457,8 @@ public Object setValue(Object value) { name = input.nextName(); if (ALWAYS_MATCH.equals(name)) { return input.read(MAP_TYPE); - } else { - input.skipValue(); } + input.skipValue(); } input.endObject(); } else { @@ -470,7 +469,7 @@ public Object setValue(Object value) { return null; } - private @Nullable Collection> getFirstMatches() throws IOException { + private @Nullable Collection> getFirstMatch() throws IOException { CharSource charSource = backingStore.asByteSource().asCharSource(UTF_8); try (Reader reader = charSource.openBufferedStream(); JsonInput input = json.newInput(reader)) { @@ -483,9 +482,8 @@ public Object setValue(Object value) { name = input.nextName(); if (FIRST_MATCH.equals(name)) { return input.read(LIST_OF_MAPS_TYPE); - } else { - input.skipValue(); } + input.skipValue(); } input.endObject(); } else {