Skip to content

Commit

Permalink
Upgrade the base Embulk SPI to 0.11, embulk-util-timestamp to 0.2.2
Browse files Browse the repository at this point in the history
  • Loading branch information
@dmikurube
dmikurube committed Mar 1, 2024
1 parent 3ebb20f commit 05757c7
Show file tree
Hide file tree
Showing 6 changed files with 45 additions and 47 deletions.
20 changes: 10 additions & 10 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,9 @@ java {
}

dependencies {
compileOnly "org.embulk:embulk-api:0.10.28"
compileOnly "org.embulk:embulk-spi:0.10.28"
compileOnly "org.embulk:embulk-spi:0.11"

implementation("org.embulk:embulk-util-config:0.3.2") {
implementation("org.embulk:embulk-util-config:0.3.4") {
// They conflict with embulk-core. They are once excluded here,
// and added explicitly with versions exactly the same with embulk-core:0.10.28.
exclude group: "com.fasterxml.jackson.core", module: "jackson-annotations"
Expand All @@ -50,21 +49,22 @@ dependencies {
// and added explicitly with versions exactly the same with embulk-core:0.10.28.
implementation "com.fasterxml.jackson.core:jackson-annotations:2.6.7"
implementation "com.fasterxml.jackson.core:jackson-core:2.6.7"
implementation "com.fasterxml.jackson.core:jackson-databind:2.6.7"
implementation "com.fasterxml.jackson.core:jackson-databind:2.6.7.5"
implementation "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7"
implementation "javax.validation:validation-api:1.1.0.Final"

implementation "org.embulk:embulk-base-restclient:0.10.1"
implementation "org.embulk:embulk-util-retryhelper-jetty92:0.8.2"

implementation "org.embulk:embulk-util-timestamp:0.2.1"
implementation "org.embulk:embulk-util-timestamp:0.2.2"

testImplementation "junit:junit:4.13.2"
testImplementation "org.embulk:embulk-core:0.10.28"
testImplementation "org.embulk:embulk-core:0.10.28:tests"
testImplementation "org.embulk:embulk-standards:0.10.28"
testImplementation "org.embulk:embulk-junit4:0.10.28"
testImplementation "org.embulk:embulk-deps:0.10.28"
testImplementation "org.embulk:embulk-core:0.11.2"
testImplementation "org.embulk:embulk-deps:0.11.2"
testImplementation "org.embulk:embulk-junit4:0.11.2"
testImplementation "org.embulk:embulk-input-file:0.11.0"
testImplementation "org.embulk:embulk-parser-csv:0.11.4"
testImplementation "com.google.guava:guava:18.0"
}

embulkPlugin {
Expand Down
15 changes: 7 additions & 8 deletions gradle.lockfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,21 @@
# This file is expected to be part of source control.
com.fasterxml.jackson.core:jackson-annotations:2.6.7=compileClasspath,runtimeClasspath
com.fasterxml.jackson.core:jackson-core:2.6.7=compileClasspath,runtimeClasspath
com.fasterxml.jackson.core:jackson-databind:2.6.7=compileClasspath,runtimeClasspath
com.fasterxml.jackson.core:jackson-databind:2.6.7.5=compileClasspath,runtimeClasspath
com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7=compileClasspath,runtimeClasspath
javax.validation:validation-api:1.1.0.Final=compileClasspath,runtimeClasspath
org.eclipse.jetty:jetty-client:9.2.14.v20151106=compileClasspath,runtimeClasspath
org.eclipse.jetty:jetty-http:9.2.14.v20151106=compileClasspath,runtimeClasspath
org.eclipse.jetty:jetty-io:9.2.14.v20151106=compileClasspath,runtimeClasspath
org.eclipse.jetty:jetty-util:9.2.14.v20151106=compileClasspath,runtimeClasspath
org.embulk:embulk-api:0.10.28=compileClasspath
org.embulk:embulk-base-restclient:0.10.1=compileClasspath,runtimeClasspath
org.embulk:embulk-spi:0.10.28=compileClasspath
org.embulk:embulk-util-config:0.3.2=compileClasspath,runtimeClasspath
org.embulk:embulk-spi:0.11=compileClasspath
org.embulk:embulk-util-config:0.3.4=compileClasspath,runtimeClasspath
org.embulk:embulk-util-json:0.1.1=compileClasspath,runtimeClasspath
org.embulk:embulk-util-retryhelper-jetty92:0.8.2=compileClasspath,runtimeClasspath
org.embulk:embulk-util-retryhelper:0.8.2=compileClasspath,runtimeClasspath
org.embulk:embulk-util-rubytime:0.3.2=runtimeClasspath
org.embulk:embulk-util-timestamp:0.2.1=compileClasspath,runtimeClasspath
org.msgpack:msgpack-core:0.8.11=compileClasspath
org.slf4j:slf4j-api:1.7.30=compileClasspath
org.embulk:embulk-util-rubytime:0.3.3=compileClasspath,runtimeClasspath
org.embulk:embulk-util-timestamp:0.2.2=compileClasspath,runtimeClasspath
org.msgpack:msgpack-core:0.8.24=compileClasspath
org.slf4j:slf4j-api:2.0.7=compileClasspath
empty=
5 changes: 2 additions & 3 deletions src/test/java/org/embulk/output/elasticsearch/ElasticsearchTestUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.embulk.EmbulkTestRuntime;
import org.embulk.config.ConfigSource;
import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask;
import org.embulk.spi.Schema;
Expand Down Expand Up @@ -99,9 +98,9 @@ public ConfigSource config()
.set("maximum_retries", 2);
}

public ConfigSource oldParserConfig(final EmbulkTestRuntime runtime)
public ConfigSource oldParserConfig()
{
return runtime.getExec().newConfigSource()
return CONFIG_MAPPER_FACTORY.newConfigSource()
.set("parser", parserConfig(schemaConfig()))
.getNested("parser");
}
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/org/embulk/output/elasticsearch/TestElasticsearchHttpClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,12 @@
package org.embulk.output.elasticsearch;

import org.eclipse.jetty.http.HttpMethod;
import org.embulk.EmbulkTestRuntime;
import org.embulk.config.ConfigException;
import org.embulk.config.ConfigSource;
import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask;
import org.embulk.spi.Exec;
import org.embulk.spi.time.Timestamp;
import org.embulk.test.EmbulkTestRuntime;
import org.embulk.util.config.ConfigMapper;
import org.embulk.util.config.ConfigMapperFactory;
import org.junit.Before;
Expand Down
32 changes: 16 additions & 16 deletions src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,22 +19,21 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.Lists;
import org.eclipse.jetty.http.HttpMethod;
import org.embulk.EmbulkTestRuntime;
import org.embulk.config.ConfigException;
import org.embulk.config.ConfigSource;
import org.embulk.config.TaskReport;
import org.embulk.config.TaskSource;
import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.AuthMethod;
import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.Mode;
import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask;
import org.embulk.spi.Exec;
import org.embulk.spi.OutputPlugin;
import org.embulk.spi.Page;
import org.embulk.spi.PageTestUtils;
import org.embulk.spi.Schema;
import org.embulk.spi.TransactionalPageOutput;
import org.embulk.spi.time.Timestamp;
import org.embulk.standards.CsvParserPlugin;
import org.embulk.parser.csv.CsvParserPlugin;
import org.embulk.test.EmbulkTestRuntime;
import org.embulk.test.PageTestUtils;
import org.embulk.util.config.ConfigMapper;
import org.junit.Before;
import org.junit.Rule;
Expand All @@ -57,6 +56,7 @@

public class TestElasticsearchOutputPlugin
{
private static final ConfigMapperFactory CONFIG_MAPPER_FACTORY = ElasticsearchOutputPlugin.CONFIG_MAPPER_FACTORY;
private static final ConfigMapper CONFIG_MAPPER = ElasticsearchOutputPlugin.CONFIG_MAPPER;

@Rule
Expand Down Expand Up @@ -85,7 +85,7 @@ public void testDefaultValues()
@Test
public void testDefaultValuesNull()
{
final ConfigSource config = runtime.getExec().newConfigSource()
final ConfigSource config = CONFIG_MAPPER_FACTORY.newConfigSource()
.set("in", utils.inputConfig())
.set("parser", utils.parserConfig(utils.schemaConfig()))
.set("type", "elasticsearch")
Expand All @@ -98,14 +98,14 @@ public void testDefaultValuesNull()
.set("bulk_size", ES_BULK_SIZE)
.set("concurrent_requests", ES_CONCURRENT_REQUESTS
);
Schema schema = utils.oldParserConfig(runtime).loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
Schema schema = CONFIG_MAPPER.map(utils.oldParserConfig(), CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
try {
plugin.transaction(config, schema, 0, new OutputPlugin.Control()
{
@Override
public List<TaskReport> run(TaskSource taskSource)
{
return Lists.newArrayList(Exec.newTaskReport());
return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport());
}
});
}
Expand All @@ -120,13 +120,13 @@ public List<TaskReport> run(TaskSource taskSource)
public void testTransaction()
{
ConfigSource config = utils.config();
Schema schema = utils.oldParserConfig(runtime).loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
Schema schema = CONFIG_MAPPER.map(utils.oldParserConfig(), CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
plugin.transaction(config, schema, 0, new OutputPlugin.Control()
{
@Override
public List<TaskReport> run(TaskSource taskSource)
{
return Lists.newArrayList(Exec.newTaskReport());
return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport());
}
});
// no error happens
Expand All @@ -136,14 +136,14 @@ public List<TaskReport> run(TaskSource taskSource)
public void testResume()
{
ConfigSource config = utils.config();
Schema schema = utils.oldParserConfig(runtime).loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
Schema schema = CONFIG_MAPPER.map(utils.oldParserConfig(), CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
final PluginTask task = CONFIG_MAPPER.map(config, PluginTask.class);
plugin.resume(task.dump(), schema, 0, new OutputPlugin.Control()
{
@Override
public List<TaskReport> run(TaskSource taskSource)
{
return Lists.newArrayList(Exec.newTaskReport());
return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport());
}
});
}
Expand All @@ -152,23 +152,23 @@ public List<TaskReport> run(TaskSource taskSource)
public void testCleanup()
{
ConfigSource config = utils.config();
Schema schema = utils.oldParserConfig(runtime).loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
Schema schema = CONFIG_MAPPER.map(utils.oldParserConfig(), CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
final PluginTask task = CONFIG_MAPPER.map(config, PluginTask.class);
plugin.cleanup(task.dump(), schema, 0, Arrays.asList(Exec.newTaskReport()));
plugin.cleanup(task.dump(), schema, 0, Arrays.asList(CONFIG_MAPPER_FACTORY.newTaskReport()));
// no error happens
}

@Test
public void testOutputByOpen() throws Exception
{
ConfigSource config = utils.config();
Schema schema = utils.oldParserConfig(runtime).loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
Schema schema = CONFIG_MAPPER.map(utils.oldParserConfig(), CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
final PluginTask task = CONFIG_MAPPER.map(config, PluginTask.class);
plugin.transaction(config, schema, 0, new OutputPlugin.Control() {
@Override
public List<TaskReport> run(TaskSource taskSource)
{
return Lists.newArrayList(Exec.newTaskReport());
return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport());
}
});
TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
Expand Down Expand Up @@ -214,7 +214,7 @@ public List<TaskReport> run(TaskSource taskSource)
public void testOpenAbort()
{
ConfigSource config = utils.config();
Schema schema = utils.oldParserConfig(runtime).loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
Schema schema = CONFIG_MAPPER.map(utils.oldParserConfig(), CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
final PluginTask task = CONFIG_MAPPER.map(config, PluginTask.class);
TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
output.abort();
Expand Down
18 changes: 9 additions & 9 deletions src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPluginJSON.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.Lists;
import org.eclipse.jetty.http.HttpMethod;
import org.embulk.EmbulkTestRuntime;
import org.embulk.config.ConfigException;
import org.embulk.config.ConfigSource;
import org.embulk.config.TaskReport;
Expand All @@ -30,9 +29,10 @@
import org.embulk.spi.Exec;
import org.embulk.spi.OutputPlugin;
import org.embulk.spi.Page;
import org.embulk.spi.PageTestUtils;
import org.embulk.spi.Schema;
import org.embulk.spi.TransactionalPageOutput;
import org.embulk.test.EmbulkTestRuntime;
import org.embulk.test.PageTestUtils;
import org.embulk.util.config.ConfigMapper;
import org.embulk.util.config.ConfigMapperFactory;
import org.junit.Before;
Expand Down Expand Up @@ -87,7 +87,7 @@ public void testTransaction()
@Override
public List<TaskReport> run(TaskSource taskSource)
{
return Lists.newArrayList(Exec.newTaskReport());
return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport());
}
});
// no error happens
Expand All @@ -104,7 +104,7 @@ public void testResume()
@Override
public List<TaskReport> run(TaskSource taskSource)
{
return Lists.newArrayList(Exec.newTaskReport());
return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport());
}
});
}
Expand All @@ -115,7 +115,7 @@ public void testCleanup()
ConfigSource config = utils.configJSON();
Schema schema = utils.JSONSchema();
final PluginTask task = CONFIG_MAPPER.map(config, PluginTask.class);
plugin.cleanup(task.dump(), schema, 0, Arrays.asList(Exec.newTaskReport()));
plugin.cleanup(task.dump(), schema, 0, Arrays.asList(CONFIG_MAPPER_FACTORY.newTaskReport()));
// no error happens
}

Expand All @@ -129,12 +129,12 @@ public void testOutputByOpen() throws Exception
@Override
public List<TaskReport> run(TaskSource taskSource)
{
return Lists.newArrayList(Exec.newTaskReport());
return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport());
}
});
TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);

List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, 1L, 32864L, "2015-01-27 19:23:49", "2015-01-27", true, 123.45, "embulk");
List<Page> pages = PageTestUtils.buildPage(Exec.getBufferAllocator(), schema, 1L, 32864L, "2015-01-27 19:23:49", "2015-01-27", true, 123.45, "embulk");
assertThat(pages.size(), is(1));
for (Page page : pages) {
output.add(page);
Expand Down Expand Up @@ -182,12 +182,12 @@ public void testOutputByOpenWithNulls() throws Exception
@Override
public List<TaskReport> run(TaskSource taskSource)
{
return Lists.newArrayList(Exec.newTaskReport());
return Lists.newArrayList(CONFIG_MAPPER_FACTORY.newTaskReport());
}
});
TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);

List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, 2L, null, null, "2015-01-27", true, 123.45, "embulk");
List<Page> pages = PageTestUtils.buildPage(Exec.getBufferAllocator(), schema, 2L, null, null, "2015-01-27", true, 123.45, "embulk");
assertThat(pages.size(), is(1));
for (Page page : pages) {
output.add(page);
Expand Down

0 comments on commit 05757c7

Please sign in to comment.