{"id":3175,"date":"2023-02-06T18:00:12","date_gmt":"2023-02-06T09:00:12","guid":{"rendered":"https:\/\/blog.since2020.jp\/?p=3175"},"modified":"2023-08-09T11:45:14","modified_gmt":"2023-08-09T02:45:14","slug":"robots-txt_check","status":"publish","type":"post","link":"https:\/\/since2020.jp\/media\/robots-txt_check\/","title":{"rendered":"\u3010\u30b9\u30af\u30ec\u30a4\u30d4\u30f3\u30b0\u3011\u7c21\u5358\u306arobots.txt\u78ba\u8a8d\u65b9\u6cd5\uff5e\u7dcf\u52d9\u7701\u3068\u56fd\u7acb\u56fd\u4f1a\u56f3\u66f8\u9928\u306e\u300crobots.txt\u300d\u3092\u8abf\u3079\u3066\u307f\u305f"},"content":{"rendered":"\n

\u3010\u30b9\u30af\u30ec\u30a4\u30d4\u30f3\u30b0\u3011\u7c21\u5358\u306arobots.txt\u78ba\u8a8d\u65b9\u6cd5\uff5e\u7dcf\u52d9\u7701\u3068\u56fd\u7acb\u56fd\u4f1a\u56f3\u66f8\u9928\u306e\u300crobots.txt\u300d\u3092\u8abf\u3079\u3066\u307f\u305f<\/p>\n\n\n

robots.txt\u3092\u898b\u308b\u306b\u306f\u3053\u308c\u3060\u3051<\/h2>\n

Python\u3067\u30b9\u30af\u30ec\u30a4\u30d4\u30f3\u30b0\u3092\u52c9\u5f37\u3057\u3066\u3044\u308b\u3068\u3001\u300c\u5b9f\u969b\u306b\u30b9\u30af\u30ec\u30a4\u30d4\u30f3\u30b0\u3057\u305f\u3044\u6642\u306f\u3007\u3007\u3092\u78ba\u8a8d\u3057\u3066\u306d\u300d\u3068\u66f8\u3044\u3066\u3042\u308b\u3053\u3068\u304c\u591a\u3044\u3068\u601d\u3044\u307e\u3059\u3002<\/p>\n

\u305d\u306e\u4e2d\u306e\u4e00\u3064\u306brobots.txt<\/strong>\u304c\u3042\u308a\u307e\u3059\u304c\u3001\u30d7\u30ed\u30b0\u30e9\u30e0\u3067\u5224\u5b9a\u3057\u305f\u308a\u3001\u8aac\u660e\u304c\u521d\u5fc3\u8005\u306b\u89aa\u5207\u3067\u306f\u306a\u304b\u3063\u305f\u308a\u3059\u308b\u30b5\u30a4\u30c8\u304c\u591a\u304b\u3063\u305f\u306e\u3067\u3001\u7c21\u5358\u306b\u8abf\u3079\u308b\u65b9\u6cd5\u3092\u5171\u6709\u3057\u307e\u3059\u3002<\/p>\n

robots.txt\u3092\u898b\u308b\u306b\u306f\u3001\u78ba\u8a8d\u3057\u305f\u3044\u30b5\u30a4\u30c8\u3078\u98db\u3093\u3067\u3001\u30db\u30fc\u30e0\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\u30fc\u306b**\/robots.txt**\u3092\u66f8\u304f\u3060\u3051\u3067\u3059\u3002<\/p>\n

\u300c\u30db\u30fc\u30e0\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\u30fc\uff1f\u3001\/robots.txt\u3092\u66f8\u304f\uff1f\u300d\u3068\u3044\u3046\u65b9\u306e\u305f\u3081\u306b\u5b9f\u969b\u306b\u3084\u3063\u3066\u307f\u307e\u3059\u3002<\/p>\n\n

\u624b\u98061\uff09\u78ba\u8a8d\u3057\u305f\u3044\u30b5\u30a4\u30c8\u3078\u79fb\u52d5<\/h2>\n

\"\"<\/p>\n

\u4eca\u56de\u306f\u3001\u5f0a\u793e\u30b5\u30a4\u30c8\u306erobots.txt\u3092\u898b\u3066\u307f\u308b\u3053\u3068\u306b\u3057\u307e\u3059\u3002<\/p>\n

\u30b5\u30a4\u30c8URL\u306f\u3001[https:\/\/blog.since2020.jp\/<\/a>]\u306a\u306e\u3067\u3001\u305d\u3053\u3078\u79fb\u52d5\u3057\u307e\u3059<\/p>\n\n

\u624b\u98062\uff09\u30b5\u30a4\u30c8\u4e0a\u90e8\u306eURL\u304c\u66f8\u3044\u3066\u3042\u308b\u5834\u6240\u3092\u30af\u30ea\u30c3\u30af<\/h2>\n

\"\"<\/p>\n

\u30d6\u30e9\u30a6\u30b6\u306b\u3088\u3063\u3066\u5834\u6240\u304c\u82e5\u5e72\u7570\u306a\u308b\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u304c\u3001URL\u3092\u66f8\u3044\u3066\u3042\u308b\u5834\u6240\u3092\u30af\u30ea\u30c3\u30af\u3059\u308b\u3068\u7de8\u96c6\u3067\u304d\u308b\u3088\u3046\u306b\u306a\u308a\u307e\u3059\u3002<\/p>\n

\u3061\u306a\u307f\u306b\u3001https:\/\/\u306e\u6b21\u306b\u6765\u308b**blog.since2020.jp** \u304c\u30db\u30fc\u30e0\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\u306b\u76f8\u5f53\u3057\u307e\u3059\u3002<\/p>\n\n

\u624b\u98063\uff09URL\u672b\u5c3e\u306b\u300c\/robots.txt\u300d \u3092\u8ffd\u8a18\u3059\u308b<\/h2>\n

\"\"<\/p>\n

\u305d\u3053\u3067\u3001\/robots.txt<\/strong> \u3092blog.since2020.jp\u306e\u4e0b\u306b\u6253\u3061\u8fbc\u307f\u307e\u3059\u3002<\/p>\n

\u305d\u3057\u3066\u30a8\u30f3\u30bf\u30fc\u30ad\u30fc\u3092\u62bc\u3059\u3068robots.txt\u306e\u4e2d\u8eab\u304c\u8868\u793a\u3055\u308c\u307e\u3057\u305f\uff01<\/p>\n\n

\u8272\u3005\u306a\u30b5\u30a4\u30c8\u306erobots.txt\u3092\u898b\u3066\u307f\u3088\u30461_\u7dcf\u52d9\u7701Web\u30b5\u30a4\u30c8<\/h2>\n

\"\"<\/p>\n

\u3053\u3053\u3067\u3044\u304f\u3064\u304b\u306e\u30b5\u30a4\u30c8\u306erobots.txt\u3092\u898b\u3066\u307f\u307e\u3057\u3087\u3046<\/p>\n

\u25cf\u7dcf\u52d9\u7701<\/p>\n

\u7dcf\u52d9\u7701\u516c\u5f0f\u30b5\u30a4\u30c8[https:\/\/www.soumu.go.jp\/<\/a>]\u3092\u307f\u3066\u307f\u307e\u3057\u3087\u3046\u3002<\/p>\n

https:\/\/www.soumu.go.jp\/ \u306brobots.txt\u3092\u3064\u3051\u308b\u3068\u3001\u4e0b\u8a18\u753b\u50cf\u304c\u8868\u793a\u3055\u308c\u307e\u3057\u305f\u3002<\/p>\n

ia_archiver<\/strong>\u3068\u3044\u3046\u30e6\u30fc\u30b6\u30fc\u30a8\u30fc\u30b8\u30a7\u30f3\u30c8\uff08UA\uff09\u304c\u30da\u30fc\u30b8\u3092\u30af\u30ed\u30fc\u30eb\u3059\u308b\u3053\u3068\u3092\u7981\u6b62\u3057\u3066\u3044\u307e\u3059\u3002<\/p>\n

ia_archiver\u306f\u3001web\u30da\u30fc\u30b8\u306e\u66f4\u65b0\u8a18\u9332\u3092\u6b8b\u3059\u305f\u3081\u306b\u4f7f\u308f\u308c\u3066\u3044\u308b\u30af\u30ed\u30fc\u30e9\u30fc\u3067\u3042\u308a\u3001\u5834\u5408\u306b\u3088\u3063\u3066\u306f\u30b5\u30a4\u30c8\u4e0a\u306b\u30ea\u30f3\u30af\u3055\u308c\u3066\u3044\u306a\u3044\u3001\u672c\u6765\u98db\u3079\u306a\u3044\u306f\u305a\u306e\u30da\u30fc\u30b8\u306e\u5c65\u6b74\u3082\u53d6\u5f97\u3059\u308b\u3053\u3068\u304c\u3042\u308a\u307e\u3059\u3002<\/p>\n

ia_archiver\u3092\u62d2\u5426\u3059\u308b\u610f\u56f3\u306f\u8272\u3005\u3042\u308a\u307e\u3059\u304c\u3001\u30da\u30fc\u30b8\u95b2\u89a7\u8005\u306b\u898b\u305b\u308b\u610f\u56f3\u304c\u306a\u3044\u30da\u30fc\u30b8\u306e\u8a18\u9332\u3092\u53d6\u3089\u308c\u3066\u3001\u8ab0\u3067\u3082\u95b2\u89a7\u3067\u304d\u3066\u3057\u307e\u3046\u306e\u306f\u4e0d\u9069\u5207\u3060\u3068\u5224\u65ad\u3057\u305f\u306e\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u306d\u3002<\/p>\n\n

\u8272\u3005\u306a\u30b5\u30a4\u30c8\u306erobots.txt\u3092\u898b\u3066\u307f\u3088\u30462_\u56fd\u7acb\u56fd\u4f1a\u56f3\u66f8\u9928Web\u30b5\u30a4\u30c8<\/h2>\n

\"\"<\/p>\n

\u25cf\u56fd\u7acb\u56fd\u4f1a\u56f3\u66f8\u9928<\/p>\n

\u56fd\u7acb\u56fd\u4f1a\u56f3\u66f8\u9928\u306e\u516c\u5f0f\u30b5\u30a4\u30c8 [https:\/\/www.ndl.go.jp\/<\/a>]\u306erobots.txt\u306f\u3069\u3046\u3067\u3057\u3087\u3046\u304b\uff1f<\/p>\n

https:\/\/www.soumu.go.jp\/ \u306brobots.txt\u3092\u3064\u3051\u308b\u3068\u3001\u30c6\u30ad\u30b9\u30c8\u30d5\u30a1\u30a4\u30eb\u300crobots.txt\u300d\u304c\u30c0\u30a6\u30f3\u30ed\u30fc\u30c9\u3055\u308c\u307e\u3057\u305f\u3002\u30b5\u30a4\u30c8\u4e0a\u3067\u8868\u793a\u3055\u308c\u305a\u306b\u3001txt\u30d5\u30a1\u30a4\u30eb\u3068\u3057\u3066\u30c0\u30a6\u30f3\u30ed\u30fc\u30c9\u3055\u308c\u308b\u5834\u5408\u3082\u3042\u308b\u307f\u305f\u3044\u3067\u3059\u3002\u4e2d\u8eab\u3092\u898b\u3066\u307f\u308b\u3068\u3001\u5168\u30e6\u30fc\u30b6\u30fc\u304c\u3000\/jp\/publication\/jmbwl\u306e\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\u3092\u30af\u30ed\u30fc\u30eb\u3059\u308b\u3053\u3068\u3092\u7981\u6b62\u3057\u3066\u3044\u307e\u3059\uff08\u4e0b\u8a18\u753b\u50cf\u3092\u53c2\u7167\uff09\u3002<\/p>\n

\u305d\u308c\u4ee5\u5916\u306b\u95a2\u3057\u3066\u306f\u3001robots.txt\u4e0a\u3067\u306f\u8a31\u53ef\u3055\u308c\u3066\u3044\u307e\u3059\u306d\u3002<\/p>\n\n

\u300crobots.txt\u300d\u30d5\u30a1\u30a4\u30eb\u306f\u7c21\u5358\u306b\u78ba\u8a8d\u53ef\u80fd<\/h2>\n

\u3053\u306e\u3088\u3046\u306brobots.txt\u30d5\u30a1\u30a4\u30eb\u306f\u7c21\u5358\u306b\u78ba\u8a8d\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u3001\u4e2d\u8eab\u3092\u898b\u3066\u307f\u308b\u3068\u8208\u5473\u6df1\u3044\u8a2d\u5b9a\u304c\u3055\u308c\u3066\u3044\u308b\u30b5\u30a4\u30c8\u304c\u898b\u3064\u304b\u308b\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002<\/p>\n

\u305f\u3060\u3057\u5b9f\u969b\u306b\u30d7\u30ed\u30b0\u30e9\u30e0\u3092\u8d70\u3089\u305b\u3066\u30af\u30ed\u30fc\u30eb\u3059\u308b\u969b\u306f\u3001\u5229\u7528\u898f\u7d04\u3067RPA\u3084\u30b9\u30af\u30ec\u30a4\u30d4\u30f3\u30b0\u304c\u7981\u6b62\u3055\u308c\u3066\u3044\u308b\u306e\u304b\u3092\u78ba\u8a8d\u3057\u3001\u51e6\u7406\u306btime.sleep(1)\u306a\u3069\u3092\u8ffd\u52a0\u3057\u3066\u30b5\u30fc\u30d0\u30fc\u306b\u8ca0\u8377\u3092\u304b\u3051\u306a\u3044\u3088\u3046\u306b\u3057\u307e\u3057\u3087\u3046\u3002<\/p>","protected":false},"excerpt":{"rendered":"

\u3010\u30b9\u30af\u30ec\u30a4\u30d4\u30f3\u30b0\u3011\u7c21\u5358\u306arobots.txt\u78ba\u8a8d\u65b9\u6cd5\uff5e\u7dcf\u52d9\u7701\u3068\u56fd\u7acb\u56fd\u4f1a\u56f3\u66f8\u9928\u306e\u300crobots.txt\u300d\u3092\u8abf\u3079\u3066\u307f\u305f robots.txt\u3092\u898b\u308b\u306b\u306f\u3053\u308c\u3060\u3051 Python\u3067\u30b9\u30af\u30ec\u30a4\u30d4\u30f3\u30b0\u3092\u52c9\u5f37\u3057\u3066\u3044\u308b\u3068\u3001\u300c\u5b9f\u969b\u306b\u30b9\u30af\u30ec\u30a4\u30d4 […]<\/p>\n","protected":false},"author":75,"featured_media":3123,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"content-type":"","swell_btn_cv_data":"","footnotes":"","_wp_rev_ctl_limit":""},"categories":[1247],"tags":[404,398,47,403,401,402,399,400],"class_list":["post-3175","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-business-transformation","tag-cv","tag-robots-txt","tag-seo","tag-403","tag-401","tag-402","tag-399","tag-400"],"_links":{"self":[{"href":"https:\/\/since2020.jp\/media\/wp-json\/wp\/v2\/posts\/3175","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/since2020.jp\/media\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/since2020.jp\/media\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/since2020.jp\/media\/wp-json\/wp\/v2\/users\/75"}],"replies":[{"embeddable":true,"href":"https:\/\/since2020.jp\/media\/wp-json\/wp\/v2\/comments?post=3175"}],"version-history":[{"count":0,"href":"https:\/\/since2020.jp\/media\/wp-json\/wp\/v2\/posts\/3175\/revisions"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/since2020.jp\/media\/wp-json\/wp\/v2\/media\/3123"}],"wp:attachment":[{"href":"https:\/\/since2020.jp\/media\/wp-json\/wp\/v2\/media?parent=3175"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/since2020.jp\/media\/wp-json\/wp\/v2\/categories?post=3175"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/since2020.jp\/media\/wp-json\/wp\/v2\/tags?post=3175"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}