DataX---数据同步json操作

json

适用于现代 C++ 的 JSON。

项目地址：https://gitcode.com/gh_mirrors/js/json

免费下载资源

IU菜籽U

23009人浏览 · 2016-09-29 18:12:14

IU菜籽U · 2016-09-29 18:12:14 发布

成功！mysql到本地打印控制台
{
"job": {
"content": [
{
"reader": {
"name": "mysqlreader",
"parameter": {
"username": "root",
"password": "root",
"column": ["ID","VISIT_IP"],
"connection": [
{
"table": [
"visit_log"
],
"jdbcUrl": [
"jdbc:mysql://192.168.0.197:3306/qyyx_test"
]
}
]
}
},

"writer": {
"name": "streamwriter",
"parameter": {
"print": false,
"encoding": "UTF-8"
}
}
}
]
}
}

任务启动时刻 : 2016-09-29 14:20:28
任务结束时刻 : 2016-09-29 14:20:52
任务总计耗时 : 23s
任务平均流量 : 364B/s
记录写入速度 : 3rec/s
读出记录总数 : 31
读写失败总数 : 0

任务启动时刻 : 2016-09-29 14:31:50
任务结束时刻 : 2016-09-29 14:32:01
任务总计耗时 : 10s
任务平均流量 : 364B/s
记录写入速度 : 3rec/s
读出记录总数 : 31
读写失败总数 : 0

{

"name": "mysqlreader",
"parameter": {
"username": "root",
"password": "root",
"connection": [
{
"jdbcUrl": ["jdbc:mysql://192.168.0.197:3306/qyyx_test"],

"table":[visit_log]

}
],
"where":"select ID,VISIT_IP from visit_log where ID < 9200"

}

成功！本地文件到本地文件

{
"setting": {},
"job": {
"setting": {
"speed": {
"channel": 2
}
},
"content": [
{
"reader": {
"name": "txtfilereader",
"parameter": {
"path": ["/home/haiwei.luo/case00/data"],
"encoding": "UTF-8",
"column": ["*"],
"fieldDelimiter": ","
}
},
"writer": {
"name": "txtfilewriter",
"parameter": {
"path": "/home/haiwei.luo/case00/result",
"fileName": "luohw",
"writeMode": "truncate",
"format": "yyyy-MM-dd"
}
}
}
]
}
}

成功！HDFS打印到本地控制台

{
"job": {
"setting": {
"speed": {
"channel": 3
}
},
"content": [
{
"reader": {
"name": "hdfsreader",
"parameter": {
"path": "/user/hive/warehouse/test.db/nnn/*",
"defaultFS": "hdfs://192.168.200.45:8020",
"column": ["*"],
"fileType": "orc",
"encoding": "UTF-8",
"fieldDelimiter": ","
}

},
"writer": {
"name": "streamwriter",
"parameter": {
"print": true
}
}
}
]
}
}

成功！oracle到本地文件

{
"job": {
"content": [
{
"reader": {
"name": "mysqlreader",
"parameter": {
"username": "root",
"password": "root",
"column": ["ID","VISIT_IP","BEF_URL","NOW_URL","CORP_CODE","PROMOTE_NO","PLATFORM","COOKIE_EANBLED","USER_AGENT","APP_VERSION","APP_NAME","COLOR","SCREEN","JAVA_ENABLED","JS_VERSION","SID","CREATE_BY","CREATE_TIME","MODIFY_BY","MODIFY_TIME"],
"connection": [
{
"table": [
"visit_log"
],
"jdbcUrl": [
"jdbc:mysql://192.168.0.197:3306/qyyx_test"
]
}
]
}
},

"writer": {
"name": "txtfilewriter",
"parameter": {
"path": "/opt/result/",
"fileName": "shun",
"writeMode": "truncate",
"format": "yyyy-MM-dd"
}
}
}
],
"setting": {
"speed": {
"channel": 10
}
}
}
}

失败！本地文件到HDFS
{
"setting": {},
"job": {
"setting": {
"speed": {
"channel": 2
}
},
"content": [
{
"reader": {
"name": "txtfilereader",
"parameter": {
"path": ["/opt/result"],
"encoding": "UTF-8",
"column": ["*"],
"fieldDelimiter": "\t"
}
},
"writer": {
"name": "hdfswriter",
"parameter": {
"defaultFS": "hdfs://192.168.200.45:8020",
"fileType": "text",
"path": "/user/hive/warehouse/test.db/nnn/",
"fileName": "text",
"column": ["*"],
"writeMode": "append",
"fieldDelimiter": "\t",
"compress":"NONE"
}
}
}
]
}
}

成功：orcle打印到Linux

{
"job": {
"setting": {
"speed": {
"channel": 5
}
},
"content": [
{
"reader": {
"name": "oraclereader",
"parameter": {
"username": "edw_src",
"password": "123456",
"where": "",
"connection": [
{
"querySql": [
"select * from ORDER_DETAIL_BAK"
],
"jdbcUrl": [
"jdbc:oracle:thin:@192.168.0.197:1521:orcl"
]
}
]
}
},
"writer": {
"name": "streamwriter",
"parameter": {
"visible": true,
"encoding": "UTF-8"
}
}
}
]
}
}

oracle到HDFS
失败！待修改

{
"job": {
"setting": {
"speed": {
"channel": 200
}
},
"content": [
{
"reader": {
"name": "oraclereader",
"parameter": {
"username": "edw_src",
"password": "123456",
"where": "",
"connection": [
{
"querySql": [
"select * from ORDER_DETAIL_BAK"
],
"jdbcUrl": [
"jdbc:oracle:thin:@192.168.0.197:1521:orcl"
]
}
]
}
},

"writer": {
"name": "hdfswriter",
"parameter": {
"defaultFS": "hdfs://192.168.200.45:8020",
"fileType": "text",
"path": "/user/hive/warehouse/test.db/nnn/",
"fileName": "text",
"column": ["*"],
"writeMode": "append",
"fieldDelimiter": "\t",
"compress":"NONE"
}
}

}
]
}
}

成功！oracle到本地文件
{
"job": {
"setting": {
"speed": {
"channel": 200
}
},
"content": [
{
"reader": {
"name": "oraclereader",
"parameter": {
"username": "edw_src",
"password": "123456",
"where": "",
"connection": [
{
"querySql": [
"select * from ORDER_DETAIL_BAK"
],
"jdbcUrl": [
"jdbc:oracle:thin:@192.168.0.197:1521:orcl"
]
}
]
}
},

"writer": {
"name": "txtfilewriter",
"parameter": {
"path": "/opt/result/",
"fileName": "shun",
"writeMode": "truncate",
"format": "yyyy-MM-dd"
}
}

}
]
}
}

channel：5

数据量：8906745556=8.3GB
2016-09-29 17:25:39.146 [job-0] INFO JobContainer - PerfTrace not enable!
2016-09-29 17:25:39.146 [job-0] INFO StandAloneJobContainerCommunicator - Total 20238811 records, 4580007717 bytes | Speed 4.65MB/s, 21530 records/s | Error 0 records, 0 bytes | All Task WaitWriterTime 2.157s | All Task WaitReaderTime 758.762s | Percentage 100.00%
2016-09-29 17:25:39.147 [job-0] INFO JobContainer -
任务启动时刻 : 2016-09-29 17:09:58
任务结束时刻 : 2016-09-29 17:25:39
任务总计耗时 : 940s
任务平均流量 : 4.65MB/s
记录写入速度 : 21530rec/s
读出记录总数 : 20238811
读写失败总数 : 0

channel：200
数据量：8906745556=8.3GB
2016-09-29 17:44:14.648 [job-0] INFO StandAloneJobContainerCommunicator - Total 20238811 records, 4580007717 bytes | Speed 4.65MB/s, 21530 records/s | Error 0 records, 0 bytes | All Task WaitWriterTime 1.418s | All Task WaitReaderTime 759.494s | Percentage 100.00%
2016-09-29 17:44:14.649 [job-0] INFO JobContainer -
任务启动时刻 : 2016-09-29 17:28:33
任务结束时刻 : 2016-09-29 17:44:14
任务总计耗时 : 941s
任务平均流量 : 4.65MB/s
记录写入速度 : 21530rec/s
读出记录总数 : 20238811
读写失败总数 : 0

GitHub 加速计划 / js / json

下载

适用于现代 C++ 的 JSON。

最近提交(Master分支：5 个月前 )

34665ae6 binary -> binary_t Signed-off-by: Robert Chisholm <robert.chisholm@sheffield.ac.uk> 11 天前

f3dc4684 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.28.9 to 3.28.10. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/9e8d0789d4a0fa9ceb6b1738f7e269594bdd67f0...b56ba49b26e50535fa1e7f7db0f4f7b4bf65d80d) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> 18 天前