{"id":6134,"date":"2024-12-27T11:12:26","date_gmt":"2024-12-27T02:12:26","guid":{"rendered":"https:\/\/blog.since2020.jp\/?p=6134"},"modified":"2024-12-27T11:12:26","modified_gmt":"2024-12-27T02:12:26","slug":"docker%e3%81%a7%e3%83%ad%e3%83%bc%e3%82%ab%e3%83%ab%e3%81%abiceberg%e7%92%b0%e5%a2%83%e3%82%92%e6%a7%8b%e7%af%89%e3%81%99%e3%82%8b%e6%96%b9%e6%b3%95","status":"publish","type":"post","link":"https:\/\/since2020.jp\/media\/docker%e3%81%a7%e3%83%ad%e3%83%bc%e3%82%ab%e3%83%ab%e3%81%abiceberg%e7%92%b0%e5%a2%83%e3%82%92%e6%a7%8b%e7%af%89%e3%81%99%e3%82%8b%e6%96%b9%e6%b3%95\/","title":{"rendered":"Docker\u3067\u30ed\u30fc\u30ab\u30eb\u306bIceberg\u74b0\u5883\u3092\u69cb\u7bc9\u3059\u308b\u65b9\u6cd5"},"content":{"rendered":"\n

\u8fd1\u5e74\u3001Apache Iceberg\u306f\u5927\u898f\u6a21\u306a\u30c7\u30fc\u30bf\u3092\u52b9\u7387\u7684\u306b\u7ba1\u7406\u3059\u308b\u305f\u3081\u306e\u30aa\u30fc\u30d7\u30f3\u30bd\u30fc\u30b9\u306e\u30c6\u30fc\u30d6\u30eb\u30d5\u30a9\u30fc\u30de\u30c3\u30c8\u3068\u3057\u3066\u6ce8\u76ee\u3055\u308c\u3066\u3044\u307e\u3059\u3002\u7279\u306bAWS\u74b0\u5883\u3067\u306fS3\u3068Athena\u306e\u7d44\u307f\u5408\u308f\u305b\u3067\u5229\u7528\u3055\u308c\u308b\u3053\u3068\u304c\u591a\u3044\u3067\u3059\u304c\u3001\u4eca\u56de\u306f\u30ed\u30fc\u30ab\u30eb\u74b0\u5883\u3067\u624b\u8efd\u306b\u8a66\u305b\u308b\u3088\u3046\u306bDocker\u3092\u4f7f\u7528\u3057\u3066Iceberg + Spark\u74b0\u5883\u3092\u69cb\u7bc9\u3057\u3066\u307f\u307e\u3059\u3002<\/p>\n\n\n

\u306f\u3058\u3081\u306b<\/h2>\n

\u8fd1\u5e74\u3001Apache Iceberg\u306f\u5927\u898f\u6a21\u306a\u30c7\u30fc\u30bf\u3092\u52b9\u7387\u7684\u306b\u7ba1\u7406\u3059\u308b\u305f\u3081\u306e\u30aa\u30fc\u30d7\u30f3\u30bd\u30fc\u30b9\u306e\u30c6\u30fc\u30d6\u30eb\u30d5\u30a9\u30fc\u30de\u30c3\u30c8\u3068\u3057\u3066\u6ce8\u76ee\u3055\u308c\u3066\u3044\u307e\u3059\u3002\u7279\u306bAWS\u74b0\u5883\u3067\u306fS3\u3068Athena\u306e\u7d44\u307f\u5408\u308f\u305b\u3067\u5229\u7528\u3055\u308c\u308b\u3053\u3068\u304c\u591a\u3044\u3067\u3059\u304c\u3001\u4eca\u56de\u306f\u30ed\u30fc\u30ab\u30eb\u74b0\u5883\u3067\u624b\u8efd\u306b\u8a66\u305b\u308b\u3088\u3046\u306bDocker\u3092\u4f7f\u7528\u3057\u3066Iceberg + Spark\u74b0\u5883\u3092\u69cb\u7bc9\u3057\u3066\u307f\u307e\u3059\u3002<\/span><\/p>\r\n

\u5168\u4f53\u306e\u30a4\u30e1\u30fc\u30b8<\/strong><\/span><\/p>\r\n

\"\"<\/p>\r\n

<\/p>\n\n

1. \u74b0\u5883\u69cb\u7bc9<\/h2>\n

\u307e\u305a\u3001Docker Compose\u3092\u5229\u7528\u3057\u3066Iceberg + Spark\u74b0\u5883\u3092\u69cb\u7bc9\u3057\u307e\u3059\u3002<\/span><\/p>\r\ndocker-compose.yml\u306e\u4f5c\u6210<\/span><\/b>\r\n

\u4ee5\u4e0b\u306e\u5185\u5bb9\u3067docker-compose.yml<\/code>\u3092\u4f5c\u6210\u3057\u3066\u304f\u3060\u3055\u3044\u3002 \u5185\u5bb9\u306f\u516c\u5f0f\u30c9\u30ad\u30e5\u30e1\u30f3\u30c8\u3092\u305d\u306e\u307e\u307e\u5229\u7528\u3055\u305b\u3066\u3044\u305f\u3060\u304d\u307e\u3059\u3002 https:\/\/iceberg.apache.org\/spark-quickstart\/#docker-compose<\/a><\/span><\/p>\r\n

\r\n
version: \"3\"\r\n\r\nservices:\r\nspark-iceberg:\r\nimage: tabulario\/spark-iceberg\r\ncontainer_name: spark-iceberg\r\nbuild: spark\/\r\nnetworks:\r\niceberg_net:\r\ndepends_on:\r\n- rest\r\n- minio\r\nvolumes:\r\n- .\/warehouse:\/home\/iceberg\/warehouse\r\n- .\/notebooks:\/home\/iceberg\/notebooks\/notebooks\r\nenvironment:\r\n- AWS_ACCESS_KEY_ID=admin\r\n- AWS_SECRET_ACCESS_KEY=password\r\n- AWS_REGION=us-east-1\r\nports:\r\n- 8888:8888\r\n- 8080:8080\r\n- 10000:10000\r\n- 10001:10001\r\nrest:\r\nimage: apache\/iceberg-rest-fixture\r\ncontainer_name: iceberg-rest\r\nnetworks:\r\niceberg_net:\r\nports:\r\n- 8181:8181\r\nenvironment:\r\n- AWS_ACCESS_KEY_ID=admin\r\n- AWS_SECRET_ACCESS_KEY=password\r\n- AWS_REGION=us-east-1\r\n- CATALOG_WAREHOUSE=s3:\/\/warehouse\/\r\n- CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO\r\n- CATALOG_S3_ENDPOINT=http:\/\/minio:9000\r\nminio:\r\nimage: minio\/minio\r\ncontainer_name: minio\r\nenvironment:\r\n- MINIO_ROOT_USER=admin\r\n- MINIO_ROOT_PASSWORD=password\r\n- MINIO_DOMAIN=minio\r\nnetworks:\r\niceberg_net:\r\naliases:\r\n- warehouse.minio\r\nports:\r\n- 9001:9001\r\n- 9000:9000\r\ncommand: [\"server\", \"\/data\", \"--console-address\", \":9001\"]\r\nmc:\r\ndepends_on:\r\n- minio\r\nimage: minio\/mc\r\ncontainer_name: mc\r\nnetworks:\r\niceberg_net:\r\nenvironment:\r\n- AWS_ACCESS_KEY_ID=admin\r\n- AWS_SECRET_ACCESS_KEY=password\r\n- AWS_REGION=us-east-1\r\nentrypoint: >\r\n\/bin\/sh -c \"\r\nuntil (\/usr\/bin\/mc config host add minio <http:\/\/minio:9000> admin password) do echo '...waiting...' && sleep 1; done;\r\n\/usr\/bin\/mc rm -r --force minio\/warehouse;\r\n\/usr\/bin\/mc mb minio\/warehouse;\r\n\/usr\/bin\/mc policy set public minio\/warehouse;\r\ntail -f \/dev\/null\r\n\"\r\nnetworks:\r\niceberg_net:\r\ndocker-compose.yml\r\nversion: \"3\"\r\n\r\nservices:\r\nspark-iceberg:\r\nimage: tabulario\/spark-iceberg\r\ncontainer_name: spark-iceberg\r\nnetworks:\r\niceberg_net:\r\ndepends_on:\r\n- minio\r\nvolumes:\r\n- .\/warehouse:\/home\/iceberg\/warehouse\r\nenvironment:\r\n- AWS_ACCESS_KEY_ID=admin\r\n- AWS_SECRET_ACCESS_KEY=password\r\n- AWS_REGION=us-east-1\r\nports:\r\n- 10000:10000\r\n\r\nminio:\r\nimage: minio\/minio\r\ncontainer_name: minio\r\nnetworks:\r\niceberg_net:\r\nvolumes:\r\n- .\/data:\/data\r\nenvironment:\r\n- MINIO_ROOT_USER=admin\r\n- MINIO_ROOT_PASSWORD=password\r\ncommand: [\"server\", \"\/data\", \"--console-address\", \":9001\"]\r\nports:\r\n- 9000:9000\r\n- 9001:9001\r\n\r\nnetworks:\r\niceberg_net:<\/code><\/pre>\r\n<\/div>\r\n
 <\/p>\r\n\u5fc5\u8981\u306a\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\u306e\u4f5c\u6210<\/span><\/b>\r\n
\u4ee5\u4e0b\u306e\u30b3\u30de\u30f3\u30c9\u3067\u5fc5\u8981\u306a\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\u3092\u4f5c\u6210\u3057\u307e\u3059\u3002<\/span><\/p>\r\n
<\/span><\/p>\r\n
\r\n
mkdir data notebooks warehouse\r\nchmod 755 data notebooks warehouse<\/code><\/pre>\r\n<\/div>\r\n
 <\/p>\r\nDocker\u30b3\u30f3\u30c6\u30ca\u306e\u8d77\u52d5<\/span><\/b>\r\n
\u4ee5\u4e0b\u306e\u30b3\u30de\u30f3\u30c9\u3067Docker Compose\u3092\u4f7f\u7528\u3057\u3066\u30b3\u30f3\u30c6\u30ca\u3092\u8d77\u52d5\u3057\u307e\u3059\u3002<\/span><\/p>\r\n
<\/span><\/p>\r\n
\r\n
docker-compose up<\/code><\/pre>\r\n<\/div>\r\n
<\/p>\n\n
2.Spark\u30bb\u30c3\u30b7\u30e7\u30f3\u306e\u8d77\u52d5\u3068\u57fa\u672c\u64cd\u4f5c<\/h2>\n
\u30b3\u30f3\u30c6\u30ca\u304c\u8d77\u52d5\u3057\u305f\u3089\u3001Spark\u30bb\u30c3\u30b7\u30e7\u30f3\u3092\u958b\u59cb\u3057\u3066\u3044\u304d\u307e\u3059\u3002\u4ee5\u4e0b\u306eURL\u304b\u3089notebook\u74b0\u5883\u306b\u5165\u308a\u307e\u3059\u3002<\/span><\/p>\r\n
http:\/\/localhost:8888\/tree?<\/a><\/span><\/p>\r\n
\"\"<\/p>\r\n
Iceberg – Getting Started.ipynb\u3092\u30af\u30ea\u30c3\u30af\u3057\u3066\u3001\u30c1\u30e5\u30fc\u30c8\u30ea\u30a2\u30eb\u74b0\u5883\u306b\u5165\u308a\u307e\u3059<\/span><\/p>\r\n
\"\"<\/p>\r\n
\u30c1\u30e5\u30fc\u30c8\u30ea\u30a2\u30eb\u306b\u6cbf\u3063\u3066\u30c6\u30fc\u30d6\u30eb\u3092\u4f5c\u6210\u3057\u307e\u3059\u3002<\/span><\/p>\r\n
\"\"<\/p>\r\n
\u6b21\u306b\u3001MinIO\u306e\u30c0\u30c3\u30b7\u30e5\u30dc\u30fc\u30c9\u3067\u30c6\u30fc\u30d6\u30eb\u306e\u30c7\u30fc\u30bf\u30d5\u30a1\u30a4\u30eb\u304c\u4f5c\u6210\u3055\u308c\u3066\u3044\u308b\u304b\u78ba\u8a8d\u3057\u307e\u3059\u3002\u30d6\u30e9\u30a6\u30b6\u3067\u4ee5\u4e0b\u306b\u30a2\u30af\u30bb\u30b9\u3057\u3066\u304f\u3060\u3055\u3044\uff1a<\/span><\/p>\r\n
http:\/\/localhost:9001\/<\/a><\/span><\/p>\r\n
\u30e6\u30fc\u30b6\u30fc\u540d\uff1aadmin \u30d1\u30b9\u30ef\u30fc\u30c9\uff1apassword<\/span><\/p>\r\n
\"\"<\/p>\r\n
<\/span><\/p>\r\n
MinIO\u306b\u30ed\u30b0\u30a4\u30f3\u3059\u308b\u3068\u3001warehouse\u30d5\u30a9\u30eb\u30c0\u5185\u306b\u4f5c\u6210\u3055\u308c\u305f\u30c7\u30fc\u30bf\u3068\u30e1\u30bf\u30c7\u30fc\u30bf\u304c\u78ba\u8a8d\u3067\u304d\u307e\u3059\u3002<\/span><\/p>\r\n
\"\"<\/p>\r\n
<\/p>\n\n
\u6700\u5f8c\u306b<\/h2>\n
\u4eca\u56de\u306fDocker\u3092\u4f7f\u7528\u3057\u3066\u30ed\u30fc\u30ab\u30eb\u74b0\u5883\u306bApache Iceberg\u3092\u30bb\u30c3\u30c8\u30a2\u30c3\u30d7\u3057\u3001\u57fa\u672c\u7684\u306a\u64cd\u4f5c\u3092\u4f53\u9a13\u3057\u307e\u3057\u305f\u3002Iceberg\u306f\u5927\u898f\u6a21\u306a\u30c7\u30fc\u30bf\u7ba1\u7406\u306b\u304a\u3044\u3066\u975e\u5e38\u306b\u6709\u7528\u306a\u30c4\u30fc\u30eb\u3067\u3059\u306e\u3067\u3001\u5f15\u304d\u7d9a\u304d\u30af\u30e9\u30a6\u30c9\u74b0\u5883\u3084\u3055\u3089\u306b\u8907\u96d1\u306a\u5206\u6790\u306b\u6311\u6226\u3057\u3066\u307f\u3066\u304f\u3060\u3055\u3044\u3002<\/span><\/p>\r\n
<\/p>","protected":false},"excerpt":{"rendered":"
\u8fd1\u5e74\u3001Apache Iceberg\u306f\u5927\u898f\u6a21\u306a\u30c7\u30fc\u30bf\u3092\u52b9\u7387\u7684\u306b\u7ba1\u7406\u3059\u308b\u305f\u3081\u306e\u30aa\u30fc\u30d7\u30f3\u30bd\u30fc\u30b9\u306e\u30c6\u30fc\u30d6\u30eb\u30d5\u30a9\u30fc\u30de\u30c3\u30c8\u3068\u3057\u3066\u6ce8\u76ee\u3055\u308c\u3066\u3044\u307e\u3059\u3002\u7279\u306bAWS\u74b0\u5883\u3067\u306fS3\u3068Athena\u306e\u7d44\u307f\u5408\u308f\u305b\u3067\u5229\u7528\u3055\u308c\u308b\u3053\u3068\u304c\u591a\u3044\u3067\u3059\u304c\u3001\u4eca\u56de\u306f\u30ed […]<\/p>\n","protected":false},"author":87,"featured_media":6141,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"content-type":"","swell_btn_cv_data":"","footnotes":"","_wp_rev_ctl_limit":""},"categories":[1249],"tags":[810,847,636,848,489],"class_list":["post-6134","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-knowledge","tag-apache-iceberg","tag-minio","tag-spark","tag-848","tag-489"],"_links":{"self":[{"href":"https:\/\/since2020.jp\/media\/wp-json\/wp\/v2\/posts\/6134","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/since2020.jp\/media\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/since2020.jp\/media\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/since2020.jp\/media\/wp-json\/wp\/v2\/users\/87"}],"replies":[{"embeddable":true,"href":"https:\/\/since2020.jp\/media\/wp-json\/wp\/v2\/comments?post=6134"}],"version-history":[{"count":0,"href":"https:\/\/since2020.jp\/media\/wp-json\/wp\/v2\/posts\/6134\/revisions"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/since2020.jp\/media\/wp-json\/wp\/v2\/media\/6141"}],"wp:attachment":[{"href":"https:\/\/since2020.jp\/media\/wp-json\/wp\/v2\/media?parent=6134"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/since2020.jp\/media\/wp-json\/wp\/v2\/categories?post=6134"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/since2020.jp\/media\/wp-json\/wp\/v2\/tags?post=6134"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}