diff --git a/.jekyll-cache/Jekyll/Cache/Jekyll--Cache/b7/9606fb3afea5bd1609ed40b622142f1c98125abcfe89a76a661b0e8e343910 b/.jekyll-cache/Jekyll/Cache/Jekyll--Cache/b7/9606fb3afea5bd1609ed40b622142f1c98125abcfe89a76a661b0e8e343910
index 34fe350..cde8acd 100644
--- a/.jekyll-cache/Jekyll/Cache/Jekyll--Cache/b7/9606fb3afea5bd1609ed40b622142f1c98125abcfe89a76a661b0e8e343910
+++ b/.jekyll-cache/Jekyll/Cache/Jekyll--Cache/b7/9606fb3afea5bd1609ed40b622142f1c98125abcfe89a76a661b0e8e343910
@@ -1 +1 @@
-I"v{"source"=>"/home/coder/share/CoderSherlock.github.io", "destination"=>"/home/coder/share/CoderSherlock.github.io/_site", "collections_dir"=>"", "cache_dir"=>".jekyll-cache", "plugins_dir"=>"_plugins", "layouts_dir"=>"_layouts", "data_dir"=>"_data", "includes_dir"=>"_includes", "collections"=>{"posts"=>{"output"=>true, "permalink"=>"/posts/:title"}}, "safe"=>false, "include"=>[".htaccess"], "exclude"=>["CHANGELOG.md", "HOW_TO_RELEASE.md", "Gemfile", "Gemfile.lock", "LICENSE", "README-*.md", "README.md", "gulpfile.js", "jekyll-text-theme.gemspec", "package-lock.json", "package.json", "/docs", "/node_modules", "/screenshots", "/test", "/vendor", "configure.sh", ".sass-cache", ".jekyll-cache", "gemfiles", "node_modules", "vendor/bundle/", "vendor/cache/", "vendor/gems/", "vendor/ruby/"], "keep_files"=>[".git", ".svn"], "encoding"=>"utf-8", "markdown_ext"=>"markdown,mkdown,mkdn,mkd,md", "strict_front_matter"=>false, "show_drafts"=>nil, "limit_posts"=>0, "future"=>false, "unpublished"=>false, "whitelist"=>[], "plugins"=>["jekyll-feed", "jekyll-paginate", "jekyll-sitemap", "jemoji", "jekyll-last-modified-at", "jekyll-seo-tag"], "markdown"=>"kramdown", "highlighter"=>"rouge", "lsi"=>false, "excerpt_separator"=>"<!--more-->", "incremental"=>false, "detach"=>false, "port"=>"4000", "host"=>"127.0.0.1", "baseurl"=>"", "show_dir_listing"=>false, "permalink"=>"/posts/:title", "paginate_path"=>"/page:num", "timezone"=>"America/New_York", "quiet"=>false, "verbose"=>false, "defaults"=>[{"scope"=>{"path"=>"", "type"=>"posts"}, "values"=>{"layout"=>"article", "sharing"=>true, "license"=>true, "aside"=>{"toc"=>true}, "show_edit_on_github"=>false, "show_subscribe"=>true, "pageview"=>true}}], "liquid"=>{"error_mode"=>"warn", "strict_filters"=>false, "strict_variables"=>false}, "kramdown"=>{"auto_ids"=>true, "toc_levels"=>[1, 2, 3, 4, 5, 6], "entity_output"=>"as_char", "smart_quotes"=>"lsquo,rsquo,ldquo,rdquo", "input"=>"GFM", "hard_wrap"=>false, "guess_lang"=>true, "footnote_nr"=>1, "show_warnings"=>false}, "text_skin"=>"default", "highlight_theme"=>"default", "url"=>"https://blog.pengzhan.dev", "fakebaseurl"=>"", "title"=>"STSD", "description"=>"My personal blog, some contents are useful, the others are not. Just like my mediocre life.\n", "lang"=>"en", "author"=>{"type"=>nil, "name"=>"Pengzhan Hao", "url"=>nil, "avatar"=>"/static/avatar.jpg", "bio"=>nil, "email"=>"haopengzhan@gmail.com", "facebook"=>nil, "twitter"=>nil, "weibo"=>nil, "googleplus"=>nil, "telegram"=>nil, "medium"=>nil, "zhihu"=>nil, "douban"=>nil, "linkedin"=>"pengzhanhao", "github"=>"codersherlock", "npm"=>nil}, "repository"=>"CoderSherlock/CoderSherlock.github.io", "repository_tree"=>"master", "paths"=>{"root"=>nil, "home"=>nil, "archive"=>nil, "rss"=>nil}, "license"=>nil, "toc"=>{"selectors"=>nil}, "mathjax"=>nil, "mathjax_autoNumber"=>nil, "mermaid"=>nil, "chart"=>nil, "paginate"=>8, "sources"=>nil, "sharing"=>{"provider"=>false, "addthis"=>{"id"=>nil}}, "comments"=>{"provider"=>"disqus", "disqus"=>{"shortname"=>"codersherlockblog"}, "gitalk"=>{"clientID"=>nil, "clientSecret"=>nil, "repository"=>nil, "owner"=>nil, "admin"=>nil}, "valine"=>{"app_id"=>nil, "app_key"=>nil, "placeholder"=>nil, "visitor"=>nil, "meta"=>nil}}, "pageview"=>{"provider"=>false, "leancloud"=>{"app_id"=>nil, "app_key"=>nil, "app_class"=>nil}}, "search"=>{"provider"=>"default", "google"=>{"custom_search_engine_id"=>nil}}, "analytics"=>{"provider"=>"google", "google"=>{"tracking_id"=>"UA-82637164-1", "anonymize_ip"=>false}}, "jekyll_compose"=>{"default_front_matter"=>{"posts"=>{"description"=>nil, "cover"=>nil, "category"=>nil, "tags"=>nil, "published"=>false, "sitemap"=>true, "permalink"=>nil}}}, "serving"=>false}:ET
\ No newline at end of file
+I"�{"source"=>"/home/coder/share/CoderSherlock.github.io", "destination"=>"/home/coder/share/CoderSherlock.github.io/_site", "collections_dir"=>"", "cache_dir"=>".jekyll-cache", "plugins_dir"=>"_plugins", "layouts_dir"=>"_layouts", "data_dir"=>"_data", "includes_dir"=>"_includes", "collections"=>{"posts"=>{"output"=>true, "permalink"=>"/posts/:title"}}, "safe"=>false, "include"=>[".htaccess"], "exclude"=>["CHANGELOG.md", "HOW_TO_RELEASE.md", "Gemfile", "Gemfile.lock", "LICENSE", "README-*.md", "README.md", "gulpfile.js", "jekyll-text-theme.gemspec", "package-lock.json", "package.json", "/docs", "/node_modules", "/screenshots", "/test", "/vendor", "configure.sh", ".sass-cache", ".jekyll-cache", "gemfiles", "node_modules", "vendor/bundle/", "vendor/cache/", "vendor/gems/", "vendor/ruby/"], "keep_files"=>[".git", ".svn"], "encoding"=>"utf-8", "markdown_ext"=>"markdown,mkdown,mkdn,mkd,md", "strict_front_matter"=>false, "show_drafts"=>nil, "limit_posts"=>0, "future"=>false, "unpublished"=>false, "whitelist"=>[], "plugins"=>["jekyll-feed", "jekyll-paginate", "jekyll-sitemap", "jemoji", "jekyll-last-modified-at", "jekyll-seo-tag"], "markdown"=>"kramdown", "highlighter"=>"rouge", "lsi"=>false, "excerpt_separator"=>"<!--more-->", "incremental"=>false, "detach"=>false, "port"=>"4000", "host"=>"127.0.0.1", "baseurl"=>"", "show_dir_listing"=>false, "permalink"=>"/posts/:title", "paginate_path"=>"/page:num", "timezone"=>"America/New_York", "quiet"=>false, "verbose"=>false, "defaults"=>[{"scope"=>{"path"=>"", "type"=>"posts"}, "values"=>{"layout"=>"article", "sharing"=>true, "license"=>true, "aside"=>{"toc"=>true}, "show_edit_on_github"=>false, "show_subscribe"=>true, "pageview"=>true}}], "liquid"=>{"error_mode"=>"warn", "strict_filters"=>false, "strict_variables"=>false}, "kramdown"=>{"auto_ids"=>true, "toc_levels"=>[1, 2, 3, 4, 5, 6], "entity_output"=>"as_char", "smart_quotes"=>"lsquo,rsquo,ldquo,rdquo", "input"=>"GFM", "hard_wrap"=>false, "guess_lang"=>true, "footnote_nr"=>1, "show_warnings"=>false}, "text_skin"=>"default", "highlight_theme"=>"default", "url"=>"https://dev.pengzhan.dev", "fakebaseurl"=>"/proxy/4000", "title"=>"STSD", "description"=>"My personal blog, some contents are useful, the others are not. Just like my mediocre life.\n", "lang"=>"en", "author"=>{"type"=>nil, "name"=>"Pengzhan Hao", "url"=>nil, "avatar"=>"/static/avatar.jpg", "bio"=>nil, "email"=>"haopengzhan@gmail.com", "facebook"=>nil, "twitter"=>nil, "weibo"=>nil, "googleplus"=>nil, "telegram"=>nil, "medium"=>nil, "zhihu"=>nil, "douban"=>nil, "linkedin"=>"pengzhanhao", "github"=>"codersherlock", "npm"=>nil}, "repository"=>"CoderSherlock/CoderSherlock.github.io", "repository_tree"=>"master", "paths"=>{"root"=>nil, "home"=>nil, "archive"=>nil, "rss"=>nil}, "license"=>nil, "toc"=>{"selectors"=>nil}, "mathjax"=>nil, "mathjax_autoNumber"=>nil, "mermaid"=>nil, "chart"=>nil, "paginate"=>8, "sources"=>nil, "sharing"=>{"provider"=>false, "addthis"=>{"id"=>nil}}, "comments"=>{"provider"=>"disqus", "disqus"=>{"shortname"=>"codersherlockblog"}, "gitalk"=>{"clientID"=>nil, "clientSecret"=>nil, "repository"=>nil, "owner"=>nil, "admin"=>nil}, "valine"=>{"app_id"=>nil, "app_key"=>nil, "placeholder"=>nil, "visitor"=>nil, "meta"=>nil}}, "pageview"=>{"provider"=>false, "leancloud"=>{"app_id"=>nil, "app_key"=>nil, "app_class"=>nil}}, "search"=>{"provider"=>"default", "google"=>{"custom_search_engine_id"=>nil}}, "analytics"=>{"provider"=>"google", "google"=>{"tracking_id"=>"UA-82637164-1", "anonymize_ip"=>false}}, "jekyll_compose"=>{"default_front_matter"=>{"posts"=>{"description"=>nil, "cover"=>nil, "category"=>nil, "tags"=>nil, "published"=>false, "sitemap"=>true, "permalink"=>nil}}}, "serving"=>false}:ET
\ No newline at end of file
diff --git a/_config.yml b/_config.yml
index 9a54ca0..d644fb7 100644
--- a/_config.yml
+++ b/_config.yml
@@ -18,9 +18,9 @@
 ##############################
 text_skin: default # "default" (default), "dark", "forest", "ocean", "chocolate", "orange"
 highlight_theme: default # "default" (default), "tomorrow", "tomorrow-night", "tomorrow-night-eighties", "tomorrow-night-blue", "tomorrow-night-bright"
-url     : "https://blog.pengzhan.dev"  # "https://dev.pengzhan.dev"
+url     : "https://dev.pengzhan.dev"
 baseurl : ""
-fakebaseurl: "" # "/proxy/4000"
+fakebaseurl: "/proxy/4000"
 title   : "STSD"
 description: > # this means to ignore newlines until "Language & timezone"
     My personal blog, some contents are useful, the others are not. Just like my mediocre life.
diff --git a/_posts/2024-04-11-Debug-kubelet.md b/_posts/2024-04-11-Debug-kubelet.md
index 91a0347..3bfbd5e 100644
--- a/_posts/2024-04-11-Debug-kubelet.md
+++ b/_posts/2024-04-11-Debug-kubelet.md
@@ -27,4 +27,4 @@ TL;DR: Bumping up to level 5 would satisfy most debugging needs.
 By the time, this note was written. In `kubelet` related code, level 8 was only used in `pkg/kubelet/prober/prober_manager.go` and level 7 was only used in `pkg/kubelet/logs/container_log_manager.go`.  And there are  11 occurrences that level 6 was used, and all of them are not part of workload lifecycle related.
 
 ## Further readings
-[Inotify watcher leaks in Kubelet]
\ No newline at end of file
+[Inotify watcher leaks in Kubelet](/posts/inotify-watcher-leaks-in-kubelet.html)
\ No newline at end of file
diff --git a/_posts/2024-04-11-inotify-watcher-leaks-in-kubelet.md b/_posts/2024-04-11-inotify-watcher-leaks-in-kubelet.md
index f3a5520..3af5be2 100644
--- a/_posts/2024-04-11-inotify-watcher-leaks-in-kubelet.md
+++ b/_posts/2024-04-11-inotify-watcher-leaks-in-kubelet.md
@@ -1,12 +1,156 @@
 ---
-layout: post
 title: Inotify watcher leaks in Kubelet
-date: 2024-04-11 16:35 -0400
+date: 2024-04-18 16:35 -0400
 description: 
-cover: 
+cover: '/static/2024-04/kubelet_inotify_leak_logo.png'
 category: 
-tags: 
-published: false
+tags: ["Kubernetes", "Kubelet", "Debug", "Inotify"]
+published: true
 sitemap: true
 permalink: 
+author: Pengzhan Hao
 ---
+
+## Symptom
+Recently, I faced an issue where Kubelet on a node reported error message failed to create file descriptors.
+
+```bash
+error creating file watcher: too many open files
+error creating file watcher: no space left on device
+```
+
+After short checking, I found the node has `max_user_watches` of 10000, but the `TotalinotifyWatches` is beyond this value. (P. S still not sure why watcher can initiate more than cap). In order to find which process occupied the most watchers. I used following command[^flbit_ino] to find it out.
+
+```bash
+echo -e "COUNT\tPID\tUSER\tCOMMAND"; sudo find /proc/[0-9]*/fdinfo -type f 2>/dev/null | sudo xargs grep ^inotify 2>/dev/null | cut -d/ -f 3 | uniq -c | sort -nr | { while read -rs COUNT PID; do echo -en "$COUNT\t$PID\t"; ps -p $PID -o user=,command=; done}
+
+COUNT	PID	USER	COMMAND
+7491	8412	root /home/kubernetes/bin/kubelet --v=2 --cloud-provide=gce --experi
+2620	1	root /sbin/init
+....
+```
+
+Surprisingly, Kubelet initiated more than 7000 inotify watchers. I assumed there was an inotify leakage in Kubelet.
+## Leakage check
+
+### Clean Kubelet
+To better understand the situation, I created a clean cluster with only 1 clean node on GKE. Roughly 70 inotify watchers were there. I created a single nginx pod and the number increased by 3. Theoretically, these 3 watchers are used by Kubelet to monitor any changes on `rootfs `, `kube-api-access` and `PodSandbox`. But to verify it, we need to check more details on which inodes are monitored by Kubelet. 
+### Check inotify file descriptors
+To do so, let's take a look how to track a single inotify file descriptor. Opened processes' `fdinfo` folder, we can examine each or them to find an inotify fd. 
+
+```bash
+# Find kubelet pid
+ps -aux | grep kubelet
+KPID=2430
+
+# File the an example fd
+sudo ls /proc/2430/fdinfo
+
+0 1 10 11 12 13 14 2 3 4 5 6 7 8 9
+
+...
+
+sudo cat /proc/2430/fdinfo/8
+
+pos:	0
+flags:	02004000
+mnt_id:	15
+ino:	1057
+inotify wd:1 ino:3f327 sdev:800001 mask:fc6 ignored_mask:0 fhandle-bytes:8 fhandle-type:1 f_handle:27f30300e5059ea2
+```
+
+This is very confusing, so I rely on `man proc`[^man_proc] to understand every piece of them. In given fd, the needed information to continue sit in the last line. It's an inotify entry represents the 1 file or folder to be monitored. And the most useful data is `ino:3f327` which represents the inode number of target file (in hexadecimal). And `sdev:800001`, which represents the ID of device where the inode sit on, and it's also in hex.
+
+Using `lsblk`, I can see there's only 1 disk I'm using on the node, so finding the target file would be easy.
+
+```bash
+# Cast to decimal
+ino=3f327
+dec="$((16#${ino}))"
+
+# Find the target file
+loc="debugfs -R 'ncheck ${dec}' /dev/sda1"
+sudo eval $loc 2>/dev/null
+
+debugfd 1.46.5 (30-Dec-2021)
+Inode	Pathname
+258855	/etc/srv/kubernetes/pki/ca-certificates.crt
+```
+
+Put all processes above into one single script, I can retrieve all target files, that would help to understand if there's a real leakage. Also, I count the unique inode amount, this could also help to know which inode are monitored multiple times.
+
+```bash
+cat << EOF | sudo tee -a test.sh
+echo "kubelet pid="${PID}
+in_fds=$(find /proc/${PID}/fdinfo -type f 2>/dev/null | xargs grep ^inotify | cut -d " " -f 3 | cut -d ":" -f 2)
+echo ${in_fds}
+echo "Count: $(find /proc/${PID}/fdinfo -type f 2>/dev/null | xargs grep ^inotify | wc -l)"
+
+uniq_fds=$(echo "${in_fds[@]}" | sort | uniq)
+echo ${uniq_fds}
+
+while read -r element;
+do
+  count=$(echo "${in_fds[@]}" | grep -o "${element}" | wc -l)
+  dec="$((16#${element}))"
+  loc="debugfs -R 'ncheck ${dec}' /dev/sda1"
+  loc=$(eval $loc 2>/dev/null | tail -1 | cut -d " " -f 4)
+  printf "%-6s %-10s %-6s %s\n" "${element}" "${dec}" "${count}" "${loc}"
+done <<< "${uniq_fds}"
+EOF
+
+sudo bash test.sh
+
+kubelet pid=2430
+3f327 3f321 ...
+Count: 120
+1 10b 1259 128a ...
+1	1	72	Inode	Pathname
+10b	267	1	267	/etc/systemd/system/multi-user.target.wants/snapd.service
+...
+```
+
+The given results are consists by following parts:
+- One line for get Kubelet pid
+- One line for all target inode numbers
+- One line for tell how many unique inode (120)
+- One line of sorted target
+- Following 120 lines, each of them represents a unique inode number, its decimal number, count, another time of decimal number and the target file path.
+
+I used the same script to the problematic node, and it showed the following result. In summary, most Kubelet watchers were targeting `ino:1 `. And there are 6649 targets files, which likely to be leakage, because there were only 150 pods on this pod. Unfortunately, `debugfs` can't find any target files, so the output showed as meaningless string `"Inode Pathname"`.
+
+```
+kubelet pid=8412
+...
+Count: 7491
+...
+1	1	6649	Inode Pathname
+```
+
+### Bad apple
+Why `debugfs` can't help anymore? The reason is simple, each cgroup for a pod is using its own rootfs. This means the watcher are somehow residing on different rootfs and using independent inode index.  There are some other ways to do it, I choose the most common tool `grep` to find out.
+
+```
+sudo grep / -inum 1
+/home/kubernetes/containerized_mounter/rootfs/dev
+/home/kubernetes/containerized_mounter/rootfs/proc
+...
+/home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet/pods/5325873d-f2a0-48df-83e2-0b911df2f77f/volumes/kubernetes.io~projected/kube-api-access-227jg
+...
+/dev
+/boot/efi
+...
+```
+
+This turns things easy, because I can just use pod ID to compare between running pods to find out if there are any terminated pods are there. And it did show there's some non-existence pods still being watched somehow.
+
+## What to expect next
+
+- [How Kubelet leaked inotify watchers?]()
+- [debugfs]()
+
+## References
+
+[^flbit_ino]: [Fluentbit error "cannot adjust chunk size" on GKE](https://stackoverflow.com/a/76712244)
+[^man_proc]: [proc(5)](https://manpages.courier-mta.org/htmlman5/proc.5.html)
+[^list_ino]: [Listing the files that are being watched by `inotify` instances](https://unix.stackexchange.com/a/646113)
\ No newline at end of file
diff --git a/_site/404.html b/_site/404.html
index 0a1c79e..80f9437 100644
--- a/_site/404.html
+++ b/_site/404.html
@@ -5,7 +5,7 @@
 
 <meta name="description" content="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
 ">
-<link rel="canonical" href="https://blog.pengzhan.dev/404.html"><link rel="alternate" type="application/rss+xml" title="STSD" href="/feed.xml"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
+<link rel="canonical" href="https://dev.pengzhan.dev/proxy/4000/404.html"><link rel="alternate" type="application/rss+xml" title="STSD" href="/proxy/4000/feed.xml"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
 <link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png">
 <link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png">
 <link rel="manifest" href="/assets/site.webmanifest">
@@ -14,7 +14,7 @@
 <meta name="msapplication-TileColor" content="#da532c">
 <meta name="msapplication-config" content="/assets/browserconfig.xml">
 <meta name="theme-color" content="#ffffff">
-<link rel="stylesheet" href="/assets/css/main.css"><link rel="stylesheet" href="/static/2022-09/all.css" >
+<link rel="stylesheet" href="/proxy/4000/assets/css/main.css"><link rel="stylesheet" href="/static/2022-09/all.css" >
 
 <style>
     @import url('https://fonts.googleapis.com/css2?family=Poppins:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;0,800;0,900;1,100;1,200;1,300;1,400;1,500;1,600;1,700;1,800;1,900&family=Roboto+Slab:wght@100..900&family=ZCOOL+XiaoWei&display=swap')
@@ -27,14 +27,14 @@
 <meta property="og:locale" content="en" />
 <meta name="description" content="My personal blog, some contents are useful, the others are not. Just like my mediocre life." />
 <meta property="og:description" content="My personal blog, some contents are useful, the others are not. Just like my mediocre life." />
-<link rel="canonical" href="https://blog.pengzhan.dev/404.html" />
-<meta property="og:url" content="https://blog.pengzhan.dev/404.html" />
+<link rel="canonical" href="https://dev.pengzhan.dev/404.html" />
+<meta property="og:url" content="https://dev.pengzhan.dev/404.html" />
 <meta property="og:site_name" content="STSD" />
 <meta property="og:type" content="website" />
 <meta name="twitter:card" content="summary" />
 <meta property="twitter:title" content="Page Not Found" />
 <script type="application/ld+json">
-{"@context":"https://schema.org","@type":"WebPage","author":{"@type":"Person","name":"Pengzhan Hao"},"dateModified":"2021-10-25T14:53:15-04:00","description":"My personal blog, some contents are useful, the others are not. Just like my mediocre life.","headline":"Page Not Found","url":"https://blog.pengzhan.dev/404.html"}</script>
+{"@context":"https://schema.org","@type":"WebPage","author":{"@type":"Person","name":"Pengzhan Hao"},"dateModified":"2021-10-25T14:53:15-04:00","description":"My personal blog, some contents are useful, the others are not. Just like my mediocre life.","headline":"Page Not Found","url":"https://dev.pengzhan.dev/404.html"}</script>
 <!-- End Jekyll SEO tag -->
 
 <!-- end custom head snippets -->
@@ -271,7 +271,7 @@
         }
       },
       paths: {
-        search_js: '/assets/search.js'
+        search_js: '/proxy/4000/assets/search.js'
       }
     };
     window.TEXT_VARIABLES = TEXT_VARIABLES;
@@ -426,9 +426,9 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
 </g>
 </svg>
 -->
-          <img src="/assets/favicon-32x32.png" style="margin-right: 5px;"><a title="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
-" href="/">STSD</a></div><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></div><nav class="navigation">
-        <ul><li class="navigation__item"><a href="/archive.html">Archive</a></li><li class="navigation__item"><a href="https://pengzhan.dev">About</a></li><li><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></li></ul>
+          <img src="/proxy/4000/assets/favicon-32x32.png" style="margin-right: 5px;"><a title="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
+" href="/proxy/4000/">STSD</a></div><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></div><nav class="navigation">
+        <ul><li class="navigation__item"><a href="/proxy/4000/archive.html">Archive</a></li><li class="navigation__item"><a href="https://pengzhan.dev">About</a></li><li><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></li></ul>
       </nav></div>
   </header>
 </div><div class="page__content"><div class ="main"><div class="grid grid--reverse">
@@ -446,7 +446,7 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
             </div></div></div><div class="page__footer d-print-none">
 <footer class="footer py-4 js-page-footer">
   <div class="main"><div itemscope itemtype="http://schema.org/Person">
-      <meta itemprop="name" content="Pengzhan Hao"><meta itemprop="url" content="/"><div class="footer__author-links"><div class="author-links">
+      <meta itemprop="name" content="Pengzhan Hao"><meta itemprop="url" content="/proxy/4000/"><div class="footer__author-links"><div class="author-links">
   <ul class="menu menu--nowrap menu--inline"><li title="Send me an Email.">
       <a class="button button--circle mail-button" itemprop="email" href="mailto:haopengzhan@gmail.com" target="_blank">
         <div class="icon"><svg fill="#000000" width="24px" height="24px" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg">
diff --git a/_site/archive.html b/_site/archive.html
index 127c65b..6d378d0 100644
--- a/_site/archive.html
+++ b/_site/archive.html
@@ -5,7 +5,7 @@
 
 <meta name="description" content="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
 ">
-<link rel="canonical" href="https://blog.pengzhan.dev/archive"><link rel="alternate" type="application/rss+xml" title="STSD" href="/feed.xml"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
+<link rel="canonical" href="https://dev.pengzhan.dev/proxy/4000/archive"><link rel="alternate" type="application/rss+xml" title="STSD" href="/proxy/4000/feed.xml"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
 <link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png">
 <link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png">
 <link rel="manifest" href="/assets/site.webmanifest">
@@ -14,7 +14,7 @@
 <meta name="msapplication-TileColor" content="#da532c">
 <meta name="msapplication-config" content="/assets/browserconfig.xml">
 <meta name="theme-color" content="#ffffff">
-<link rel="stylesheet" href="/assets/css/main.css"><link rel="stylesheet" href="/static/2022-09/all.css" >
+<link rel="stylesheet" href="/proxy/4000/assets/css/main.css"><link rel="stylesheet" href="/static/2022-09/all.css" >
 
 <style>
     @import url('https://fonts.googleapis.com/css2?family=Poppins:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;0,800;0,900;1,100;1,200;1,300;1,400;1,500;1,600;1,700;1,800;1,900&family=Roboto+Slab:wght@100..900&family=ZCOOL+XiaoWei&display=swap')
@@ -27,14 +27,14 @@
 <meta property="og:locale" content="en" />
 <meta name="description" content="My personal blog, some contents are useful, the others are not. Just like my mediocre life." />
 <meta property="og:description" content="My personal blog, some contents are useful, the others are not. Just like my mediocre life." />
-<link rel="canonical" href="https://blog.pengzhan.dev/archive" />
-<meta property="og:url" content="https://blog.pengzhan.dev/archive" />
+<link rel="canonical" href="https://dev.pengzhan.dev/archive" />
+<meta property="og:url" content="https://dev.pengzhan.dev/archive" />
 <meta property="og:site_name" content="STSD" />
 <meta property="og:type" content="website" />
 <meta name="twitter:card" content="summary" />
 <meta property="twitter:title" content="STSD" />
 <script type="application/ld+json">
-{"@context":"https://schema.org","@type":"WebPage","author":{"@type":"Person","name":"Pengzhan Hao"},"dateModified":"2021-10-25T14:53:15-04:00","description":"My personal blog, some contents are useful, the others are not. Just like my mediocre life.","headline":"STSD","url":"https://blog.pengzhan.dev/archive"}</script>
+{"@context":"https://schema.org","@type":"WebPage","author":{"@type":"Person","name":"Pengzhan Hao"},"dateModified":"2021-10-25T14:53:15-04:00","description":"My personal blog, some contents are useful, the others are not. Just like my mediocre life.","headline":"STSD","url":"https://dev.pengzhan.dev/archive"}</script>
 <!-- End Jekyll SEO tag -->
 
 <!-- end custom head snippets -->
@@ -271,7 +271,7 @@
         }
       },
       paths: {
-        search_js: '/assets/search.js'
+        search_js: '/proxy/4000/assets/search.js'
       }
     };
     window.TEXT_VARIABLES = TEXT_VARIABLES;
@@ -426,9 +426,9 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
 </g>
 </svg>
 -->
-          <img src="/assets/favicon-32x32.png" style="margin-right: 5px;"><a title="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
-" href="/">STSD</a></div><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></div><nav class="navigation">
-        <ul><li class="navigation__item"><a href="/archive.html">Archive</a></li><li class="navigation__item"><a href="https://pengzhan.dev">About</a></li><li><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></li></ul>
+          <img src="/proxy/4000/assets/favicon-32x32.png" style="margin-right: 5px;"><a title="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
+" href="/proxy/4000/">STSD</a></div><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></div><nav class="navigation">
+        <ul><li class="navigation__item"><a href="/proxy/4000/archive.html">Archive</a></li><li class="navigation__item"><a href="https://pengzhan.dev">About</a></li><li><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></li></ul>
       </nav></div>
   </header>
 </div><div class="page__content"><div class ="main"><div class="grid grid--reverse">
@@ -442,7 +442,7 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
   <ul class="menu">
     <li>
       <button type="button" class="button button--secondary button--pill tag-button tag-button--all" data-encode="">
-        Show All<div class="tag-button__count">9</div>
+        Show All<div class="tag-button__count">10</div>
       </button>
     </li><li><button type="button" class="button button--pill tag-button tag-button-1" data-encode="Binghamton+university">
           <span>Binghamton university</span><div class="tag-button__count">1</div>
@@ -450,8 +450,8 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
       </li><li><button type="button" class="button button--pill tag-button tag-button-1" data-encode="Charles+proxy">
           <span>Charles proxy</span><div class="tag-button__count">1</div>
         </button>
-      </li><li><button type="button" class="button button--pill tag-button tag-button-1" data-encode="Debug">
-          <span>Debug</span><div class="tag-button__count">1</div>
+      </li><li><button type="button" class="button button--pill tag-button tag-button-2" data-encode="Debug">
+          <span>Debug</span><div class="tag-button__count">2</div>
         </button>
       </li><li><button type="button" class="button button--pill tag-button tag-button-1" data-encode="Diary">
           <span>Diary</span><div class="tag-button__count">1</div>
@@ -459,11 +459,14 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
       </li><li><button type="button" class="button button--pill tag-button tag-button-2" data-encode="Edge+computing">
           <span>Edge computing</span><div class="tag-button__count">2</div>
         </button>
-      </li><li><button type="button" class="button button--pill tag-button tag-button-1" data-encode="Kubelet">
-          <span>Kubelet</span><div class="tag-button__count">1</div>
+      </li><li><button type="button" class="button button--pill tag-button tag-button-1" data-encode="Inotify">
+          <span>Inotify</span><div class="tag-button__count">1</div>
         </button>
-      </li><li><button type="button" class="button button--pill tag-button tag-button-1" data-encode="Kubernetes">
-          <span>Kubernetes</span><div class="tag-button__count">1</div>
+      </li><li><button type="button" class="button button--pill tag-button tag-button-2" data-encode="Kubelet">
+          <span>Kubelet</span><div class="tag-button__count">2</div>
+        </button>
+      </li><li><button type="button" class="button button--pill tag-button tag-button-2" data-encode="Kubernetes">
+          <span>Kubernetes</span><div class="tag-button__count">2</div>
         </button>
       </li><li><button type="button" class="button button--pill tag-button tag-button-1" data-encode="Log">
           <span>Log</span><div class="tag-button__count">1</div>
@@ -491,24 +494,26 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
         </button>
       </li></ul>
 </div>
-<div class="js-result layout--archive__result d-none"><div class="article-list items"><section><h2 class="article-list__group-header">2024</h2><ul class="items"><li class="item" itemscope itemtype="http://schema.org/BlogPosting" data-tags="Kubernetes,Kubelet,Debug">
-        <div class="item__content"><span class="item__meta">Apr 10</span><a itemprop="headline" class="item__header" href="/posts/Debug-kubelet">Debug Kubelet</a></div>
+<div class="js-result layout--archive__result d-none"><div class="article-list items"><section><h2 class="article-list__group-header">2024</h2><ul class="items"><li class="item" itemscope itemtype="http://schema.org/BlogPosting" data-tags="Kubernetes,Kubelet,Debug,Inotify">
+        <div class="item__content"><span class="item__meta">Apr 18</span><a itemprop="headline" class="item__header" href="/proxy/4000/posts/inotify-watcher-leaks-in-kubelet">Inotify watcher leaks in Kubelet</a></div>
+      </li><li class="item" itemscope itemtype="http://schema.org/BlogPosting" data-tags="Kubernetes,Kubelet,Debug">
+        <div class="item__content"><span class="item__meta">Apr 10</span><a itemprop="headline" class="item__header" href="/proxy/4000/posts/Debug-kubelet">Debug Kubelet</a></div>
       </li></ul></section><section><h2 class="article-list__group-header">2022</h2><ul class="items"><li class="item" itemscope itemtype="http://schema.org/BlogPosting" data-tags="Xv6,Teaching,Operating+system,Binghamton+university">
-        <div class="item__content"><span class="item__meta">Feb 22</span><a itemprop="headline" class="item__header" href="/posts/cs350-labs">Labs of CS350</a></div>
+        <div class="item__content"><span class="item__meta">Feb 22</span><a itemprop="headline" class="item__header" href="/proxy/4000/posts/cs350-labs">Labs of CS350</a></div>
       </li></ul></section><section><h2 class="article-list__group-header">2021</h2><ul class="items"><li class="item" itemscope itemtype="http://schema.org/BlogPosting" data-tags="Research,Edge+computing">
-        <div class="item__content"><span class="item__meta">Oct 31</span><a itemprop="headline" class="item__header" href="/posts/eddl-how-do-we-train-on-limited-edge-devices-part2">EDDL: How do we train neural networks on limited edge devices - PART 2</a></div>
+        <div class="item__content"><span class="item__meta">Oct 31</span><a itemprop="headline" class="item__header" href="/proxy/4000/posts/eddl-how-do-we-train-on-limited-edge-devices-part2">EDDL: How do we train neural networks on limited edge devices - PART 2</a></div>
       </li><li class="item" itemscope itemtype="http://schema.org/BlogPosting" data-tags="Research,Edge+computing">
-        <div class="item__content"><span class="item__meta">Oct 13</span><a itemprop="headline" class="item__header" href="/posts/eddl-how-do-we-train-on-limited-edge-devices">EDDL: How do we train neural networks on limited edge devices - PART 1</a></div>
+        <div class="item__content"><span class="item__meta">Oct 13</span><a itemprop="headline" class="item__header" href="/proxy/4000/posts/eddl-how-do-we-train-on-limited-edge-devices">EDDL: How do we train neural networks on limited edge devices - PART 1</a></div>
       </li></ul></section><section><h2 class="article-list__group-header">2020</h2><ul class="items"><li class="item" itemscope itemtype="http://schema.org/BlogPosting" data-tags="Visualization">
-        <div class="item__content"><span class="item__meta">Sep 15</span><a itemprop="headline" class="item__header" href="/posts/generate-word-cloud-with-chinese-fenci">Generate Word Cloud Figures with Chinese-Tokenization and WordCloud python libraries</a></div>
+        <div class="item__content"><span class="item__meta">Sep 15</span><a itemprop="headline" class="item__header" href="/proxy/4000/posts/generate-word-cloud-with-chinese-fenci">Generate Word Cloud Figures with Chinese-Tokenization and WordCloud python libraries</a></div>
       </li></ul></section><section><h2 class="article-list__group-header">2017</h2><ul class="items"><li class="item" itemscope itemtype="http://schema.org/BlogPosting" data-tags="Xv6,Teaching,Operating+system">
-        <div class="item__content"><span class="item__meta">Jul 28</span><a itemprop="headline" class="item__header" href="/posts/intro-xv6">Xv6 introduction</a></div>
+        <div class="item__content"><span class="item__meta">Jul 28</span><a itemprop="headline" class="item__header" href="/proxy/4000/posts/intro-xv6">Xv6 introduction</a></div>
       </li></ul></section><section><h2 class="article-list__group-header">2016</h2><ul class="items"><li class="item" itemscope itemtype="http://schema.org/BlogPosting" data-tags="Research,Log,Miscellanies">
-        <div class="item__content"><span class="item__meta">Oct 28</span><a itemprop="headline" class="item__header" href="/posts/some-of-my-previews-exper-work">Some of my previews experiment works: 2016</a></div>
+        <div class="item__content"><span class="item__meta">Oct 28</span><a itemprop="headline" class="item__header" href="/proxy/4000/posts/some-of-my-previews-exper-work">Some of my previews experiment works: 2016</a></div>
       </li><li class="item" itemscope itemtype="http://schema.org/BlogPosting" data-tags="Network,Charles+proxy">
-        <div class="item__content"><span class="item__meta">Oct 27</span><a itemprop="headline" class="item__header" href="/posts/charles-is-not-a-good-tool">Using charles proxy to monitor mobile SSL traffics</a></div>
+        <div class="item__content"><span class="item__meta">Oct 27</span><a itemprop="headline" class="item__header" href="/proxy/4000/posts/charles-is-not-a-good-tool">Using charles proxy to monitor mobile SSL traffics</a></div>
       </li><li class="item" itemscope itemtype="http://schema.org/BlogPosting" data-tags="Diary">
-        <div class="item__content"><span class="item__meta">Oct 26</span><a itemprop="headline" class="item__header" href="/posts/welcome-to-my-blog">STSD: Stop Talking Start Doing</a></div>
+        <div class="item__content"><span class="item__meta">Oct 26</span><a itemprop="headline" class="item__header" href="/proxy/4000/posts/welcome-to-my-blog">STSD: Stop Talking Start Doing</a></div>
       </li></div>
 </div>
 </div>
@@ -658,7 +663,7 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
             </div></div></div><div class="page__footer d-print-none">
 <footer class="footer py-4 js-page-footer">
   <div class="main"><div itemscope itemtype="http://schema.org/Person">
-      <meta itemprop="name" content="Pengzhan Hao"><meta itemprop="url" content="/"><div class="footer__author-links"><div class="author-links">
+      <meta itemprop="name" content="Pengzhan Hao"><meta itemprop="url" content="/proxy/4000/"><div class="footer__author-links"><div class="author-links">
   <ul class="menu menu--nowrap menu--inline"><li title="Send me an Email.">
       <a class="button button--circle mail-button" itemprop="email" href="mailto:haopengzhan@gmail.com" target="_blank">
         <div class="icon"><svg fill="#000000" width="24px" height="24px" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg">
diff --git a/_site/assets/search.js b/_site/assets/search.js
index 796be97..ff92cac 100644
--- a/_site/assets/search.js
+++ b/_site/assets/search.js
@@ -1 +1 @@
-window.TEXT_SEARCH_DATA={'posts':[{'title':"STSD: Stop Talking Start Doing",'url':"/posts/welcome-to-my-blog"},{'title':"Using charles proxy to monitor mobile SSL traffics",'url':"/posts/charles-is-not-a-good-tool"},{'title':"Some of my previews experiment works: 2016",'url':"/posts/some-of-my-previews-exper-work"},{'title':"Xv6 introduction",'url':"/posts/intro-xv6"},{'title':"Generate Word Cloud Figures with Chinese-Tokenization and WordCloud python libraries",'url':"/posts/generate-word-cloud-with-chinese-fenci"},{'title':"EDDL: How do we train neural networks on limited edge devices - PART 1",'url':"/posts/eddl-how-do-we-train-on-limited-edge-devices"},{'title':"EDDL: How do we train neural networks on limited edge devices - PART 2",'url':"/posts/eddl-how-do-we-train-on-limited-edge-devices-part2"},{'title':"Labs of CS350",'url':"/posts/cs350-labs"},{'title':"Debug Kubelet",'url':"/posts/Debug-kubelet"}]};
+window.TEXT_SEARCH_DATA={'posts':[{'title':"STSD: Stop Talking Start Doing",'url':"/proxy/4000/posts/welcome-to-my-blog"},{'title':"Using charles proxy to monitor mobile SSL traffics",'url':"/proxy/4000/posts/charles-is-not-a-good-tool"},{'title':"Some of my previews experiment works: 2016",'url':"/proxy/4000/posts/some-of-my-previews-exper-work"},{'title':"Xv6 introduction",'url':"/proxy/4000/posts/intro-xv6"},{'title':"Generate Word Cloud Figures with Chinese-Tokenization and WordCloud python libraries",'url':"/proxy/4000/posts/generate-word-cloud-with-chinese-fenci"},{'title':"EDDL: How do we train neural networks on limited edge devices - PART 1",'url':"/proxy/4000/posts/eddl-how-do-we-train-on-limited-edge-devices"},{'title':"EDDL: How do we train neural networks on limited edge devices - PART 2",'url':"/proxy/4000/posts/eddl-how-do-we-train-on-limited-edge-devices-part2"},{'title':"Labs of CS350",'url':"/proxy/4000/posts/cs350-labs"},{'title':"Debug Kubelet",'url':"/proxy/4000/posts/Debug-kubelet"},{'title':"Inotify watcher leaks in Kubelet",'url':"/proxy/4000/posts/inotify-watcher-leaks-in-kubelet"}]};
diff --git a/_site/feed.xml b/_site/feed.xml
index 09ff5cf..d749886 100644
--- a/_site/feed.xml
+++ b/_site/feed.xml
@@ -1,5 +1,151 @@
-<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom" xml:lang="en"><generator uri="https://jekyllrb.com/" version="4.2.2">Jekyll</generator><link href="https://blog.pengzhan.dev/feed.xml" rel="self" type="application/atom+xml" /><link href="https://blog.pengzhan.dev/" rel="alternate" type="text/html" hreflang="en" /><updated>2024-04-13T17:17:15-04:00</updated><id>https://blog.pengzhan.dev/feed.xml</id><title type="html">STSD</title><subtitle>My personal blog, some contents are useful, the others are not. Just like my mediocre life.
-</subtitle><author><name>Pengzhan Hao</name><email>haopengzhan@gmail.com</email></author><entry><title type="html">Debug Kubelet</title><link href="https://blog.pengzhan.dev/posts/Debug-kubelet" rel="alternate" type="text/html" title="Debug Kubelet" /><published>2024-04-10T03:34:00-04:00</published><updated>2024-04-10T23:51:19-04:00</updated><id>https://blog.pengzhan.dev/posts/Debug-kubelet</id><content type="html" xml:base="https://blog.pengzhan.dev/posts/Debug-kubelet"><![CDATA[<h2 id="debug-logs">Debug logs</h2>
+<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom" xml:lang="en"><generator uri="https://jekyllrb.com/" version="4.2.2">Jekyll</generator><link href="https://dev.pengzhan.dev/feed.xml" rel="self" type="application/atom+xml" /><link href="https://dev.pengzhan.dev/" rel="alternate" type="text/html" hreflang="en" /><updated>2024-04-18T21:14:05-04:00</updated><id>https://dev.pengzhan.dev/feed.xml</id><title type="html">STSD</title><subtitle>My personal blog, some contents are useful, the others are not. Just like my mediocre life.
+</subtitle><author><name>Pengzhan Hao</name><email>haopengzhan@gmail.com</email></author><entry><title type="html">Inotify watcher leaks in Kubelet</title><link href="https://dev.pengzhan.dev/posts/inotify-watcher-leaks-in-kubelet" rel="alternate" type="text/html" title="Inotify watcher leaks in Kubelet" /><published>2024-04-18T16:35:00-04:00</published><updated>2024-04-11T16:57:07-04:00</updated><id>https://dev.pengzhan.dev/posts/inotify-watcher-leaks-in-kubelet</id><content type="html" xml:base="https://dev.pengzhan.dev/posts/inotify-watcher-leaks-in-kubelet"><![CDATA[<h2 id="symptom">Symptom</h2>
+<p>Recently, I faced an issue where Kubelet on a node reported error message failed to create file descriptors.</p>
+
+<div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>error creating file watcher: too many open files
+error creating file watcher: no space left on device
+</code></pre></div></div>
+
+<p>After short checking, I found the node has <code class="language-plaintext highlighter-rouge">max_user_watches</code> of 10000, but the <code class="language-plaintext highlighter-rouge">TotalinotifyWatches</code> is beyond this value. (P. S still not sure why watcher can initiate more than cap). In order to find which process occupied the most watchers. I used following command<sup id="fnref:flbit_ino" role="doc-noteref"><a href="#fn:flbit_ino" class="footnote" rel="footnote">1</a></sup> to find it out.</p>
+
+<div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">echo</span> <span class="nt">-e</span> <span class="s2">"COUNT</span><span class="se">\t</span><span class="s2">PID</span><span class="se">\t</span><span class="s2">USER</span><span class="se">\t</span><span class="s2">COMMAND"</span><span class="p">;</span> <span class="nb">sudo </span>find /proc/[0-9]<span class="k">*</span>/fdinfo <span class="nt">-type</span> f 2&gt;/dev/null | <span class="nb">sudo </span>xargs <span class="nb">grep</span> ^inotify 2&gt;/dev/null | <span class="nb">cut</span> <span class="nt">-d</span>/ <span class="nt">-f</span> 3 | <span class="nb">uniq</span> <span class="nt">-c</span> | <span class="nb">sort</span> <span class="nt">-nr</span> | <span class="o">{</span> <span class="k">while </span><span class="nb">read</span> <span class="nt">-rs</span> COUNT PID<span class="p">;</span> <span class="k">do </span><span class="nb">echo</span> <span class="nt">-en</span> <span class="s2">"</span><span class="nv">$COUNT</span><span class="se">\t</span><span class="nv">$PID</span><span class="se">\t</span><span class="s2">"</span><span class="p">;</span> ps <span class="nt">-p</span> <span class="nv">$PID</span> <span class="nt">-o</span> <span class="nv">user</span><span class="o">=</span>,command<span class="o">=</span><span class="p">;</span> <span class="k">done</span><span class="o">}</span>
+
+COUNT	PID	USER	COMMAND
+7491	8412	root /home/kubernetes/bin/kubelet <span class="nt">--v</span><span class="o">=</span>2 <span class="nt">--cloud-provide</span><span class="o">=</span>gce <span class="nt">--experi</span>
+2620	1	root /sbin/init
+....
+</code></pre></div></div>
+
+<p>Surprisingly, Kubelet initiated more than 7000 inotify watchers. I assumed there was an inotify leakage in Kubelet.</p>
+<h2 id="leakage-check">Leakage check</h2>
+
+<h3 id="clean-kubelet">Clean Kubelet</h3>
+<p>To better understand the situation, I created a clean cluster with only 1 clean node on GKE. Roughly 70 inotify watchers were there. I created a single nginx pod and the number increased by 3. Theoretically, these 3 watchers are used by Kubelet to monitor any changes on <code class="language-plaintext highlighter-rouge">rootfs </code>, <code class="language-plaintext highlighter-rouge">kube-api-access</code> and <code class="language-plaintext highlighter-rouge">PodSandbox</code>. But to verify it, we need to check more details on which inodes are monitored by Kubelet.</p>
+<h3 id="check-inotify-file-descriptors">Check inotify file descriptors</h3>
+<p>To do so, let’s take a look how to track a single inotify file descriptor. Opened processes’ <code class="language-plaintext highlighter-rouge">fdinfo</code> folder, we can examine each or them to find an inotify fd.</p>
+
+<div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c"># Find kubelet pid</span>
+ps <span class="nt">-aux</span> | <span class="nb">grep </span>kubelet
+<span class="nv">KPID</span><span class="o">=</span>2430
+
+<span class="c"># File the an example fd</span>
+<span class="nb">sudo ls</span> /proc/2430/fdinfo
+
+0 1 10 11 12 13 14 2 3 4 5 6 7 8 9
+
+...
+
+<span class="nb">sudo cat</span> /proc/2430/fdinfo/8
+
+pos:	0
+flags:	02004000
+mnt_id:	15
+ino:	1057
+inotify wd:1 ino:3f327 sdev:800001 mask:fc6 ignored_mask:0 fhandle-bytes:8 fhandle-type:1 f_handle:27f30300e5059ea2
+</code></pre></div></div>
+
+<p>This is very confusing, so I rely on <code class="language-plaintext highlighter-rouge">man proc</code><sup id="fnref:man_proc" role="doc-noteref"><a href="#fn:man_proc" class="footnote" rel="footnote">2</a></sup> to understand every piece of them. In given fd, the needed information to continue sit in the last line. It’s an inotify entry represents the 1 file or folder to be monitored. And the most useful data is <code class="language-plaintext highlighter-rouge">ino:3f327</code> which represents the inode number of target file (in hexadecimal). And <code class="language-plaintext highlighter-rouge">sdev:800001</code>, which represents the ID of device where the inode sit on, and it’s also in hex.</p>
+
+<p>Using <code class="language-plaintext highlighter-rouge">lsblk</code>, I can see there’s only 1 disk I’m using on the node, so finding the target file would be easy.</p>
+
+<div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c"># Cast to decimal</span>
+<span class="nv">ino</span><span class="o">=</span>3f327
+<span class="nv">dec</span><span class="o">=</span><span class="s2">"</span><span class="k">$((</span><span class="m">16</span><span class="c">#${ino}))"</span>
+
+<span class="c"># Find the target file</span>
+<span class="nv">loc</span><span class="o">=</span><span class="s2">"debugfs -R 'ncheck </span><span class="k">${</span><span class="nv">dec</span><span class="k">}</span><span class="s2">' /dev/sda1"</span>
+<span class="nb">sudo eval</span> <span class="nv">$loc</span> <span class="m">2</span><span class="o">&gt;/</span>dev/null
+
+debugfd <span class="m">1</span>.46.5 <span class="o">(</span><span class="m">30</span><span class="o">-</span>Dec-2021<span class="o">)</span>
+Inode	Pathname
+<span class="m">258855</span>	<span class="o">/</span>etc/srv/kubernetes/pki/ca-certificates.crt
+</code></pre></div></div>
+
+<p>Put all processes above into one single script, I can retrieve all target files, that would help to understand if there’s a real leakage. Also, I count the unique inode amount, this could also help to know which inode are monitored multiple times.</p>
+
+<div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">cat</span> <span class="o">&lt;&lt;</span> <span class="no">EOF</span><span class="sh"> | sudo tee -a test.sh
+echo "kubelet pid="</span><span class="k">${</span><span class="nv">PID</span><span class="k">}</span><span class="sh">
+in_fds=</span><span class="si">$(</span>find /proc/<span class="k">${</span><span class="nv">PID</span><span class="k">}</span>/fdinfo <span class="nt">-type</span> f 2&gt;/dev/null | xargs <span class="nb">grep</span> ^inotify | <span class="nb">cut</span> <span class="nt">-d</span> <span class="s2">" "</span> <span class="nt">-f</span> 3 | <span class="nb">cut</span> <span class="nt">-d</span> <span class="s2">":"</span> <span class="nt">-f</span> 2<span class="si">)</span><span class="sh">
+echo </span><span class="k">${</span><span class="nv">in_fds</span><span class="k">}</span><span class="sh">
+echo "Count: </span><span class="si">$(</span>find /proc/<span class="k">${</span><span class="nv">PID</span><span class="k">}</span>/fdinfo <span class="nt">-type</span> f 2&gt;/dev/null | xargs <span class="nb">grep</span> ^inotify | <span class="nb">wc</span> <span class="nt">-l</span><span class="si">)</span><span class="sh">"
+
+uniq_fds=</span><span class="si">$(</span><span class="nb">echo</span> <span class="s2">"</span><span class="k">${</span><span class="nv">in_fds</span><span class="p">[@]</span><span class="k">}</span><span class="s2">"</span> | <span class="nb">sort</span> | <span class="nb">uniq</span><span class="si">)</span><span class="sh">
+echo </span><span class="k">${</span><span class="nv">uniq_fds</span><span class="k">}</span><span class="sh">
+
+while read -r element;
+do
+  count=</span><span class="si">$(</span><span class="nb">echo</span> <span class="s2">"</span><span class="k">${</span><span class="nv">in_fds</span><span class="p">[@]</span><span class="k">}</span><span class="s2">"</span> | <span class="nb">grep</span> <span class="nt">-o</span> <span class="s2">"</span><span class="k">${</span><span class="nv">element</span><span class="k">}</span><span class="s2">"</span> | <span class="nb">wc</span> <span class="nt">-l</span><span class="si">)</span><span class="sh">
+  dec="</span><span class="k">$((</span><span class="m">16</span><span class="c">#${element}))"</span>
+  <span class="nv">loc</span><span class="o">=</span><span class="s2">"debugfs -R 'ncheck </span><span class="k">${</span><span class="nv">dec</span><span class="k">}</span><span class="s2">' /dev/sda1"</span>
+  <span class="nv">loc</span><span class="o">=</span><span class="si">$(</span><span class="nb">eval</span> <span class="nv">$loc</span> 2&gt;/dev/null | <span class="nb">tail</span> <span class="nt">-1</span> | <span class="nb">cut</span> <span class="nt">-d</span> <span class="s2">" "</span> <span class="nt">-f</span> 4<span class="si">)</span>
+  <span class="nb">printf</span> <span class="s2">"%-6s %-10s %-6s %s</span><span class="se">\n</span><span class="s2">"</span> <span class="s2">"</span><span class="k">${</span><span class="nv">element</span><span class="k">}</span><span class="s2">"</span> <span class="s2">"</span><span class="k">${</span><span class="nv">dec</span><span class="k">}</span><span class="s2">"</span> <span class="s2">"</span><span class="k">${</span><span class="nv">count</span><span class="k">}</span><span class="s2">"</span> <span class="s2">"</span><span class="k">${</span><span class="nv">loc</span><span class="k">}</span><span class="s2">"</span>
+<span class="k">done</span> <span class="o">&lt;&lt;&lt;</span> <span class="s2">"</span><span class="k">${</span><span class="nv">uniq_fds</span><span class="k">}</span><span class="s2">"</span>
+EOF
+
+<span class="nb">sudo </span>bash test.sh
+
+kubelet <span class="nv">pid</span><span class="o">=</span><span class="m">2430</span>
+<span class="m">3</span>f327 <span class="m">3</span>f321 ...
+Count: <span class="m">120</span>
+<span class="m">1</span> <span class="m">10</span>b <span class="m">1259</span> <span class="m">128</span>a ...
+<span class="m">1</span>	<span class="m">1</span>	<span class="m">72</span>	Inode	Pathname
+<span class="m">10</span>b	<span class="m">267</span>	<span class="m">1</span>	<span class="m">267</span>	<span class="o">/</span>etc/systemd/system/multi-user.target.wants/snapd.service
+...
+</code></pre></div></div>
+
+<p>The given results are consists by following parts:</p>
+<ul>
+  <li>One line for get Kubelet pid</li>
+  <li>One line for all target inode numbers</li>
+  <li>One line for tell how many unique inode (120)</li>
+  <li>One line of sorted target</li>
+  <li>Following 120 lines, each of them represents a unique inode number, its decimal number, count, another time of decimal number and the target file path.</li>
+</ul>
+
+<p>I used the same script to the problematic node, and it showed the following result. In summary, most Kubelet watchers were targeting <code class="language-plaintext highlighter-rouge">ino:1 </code>. And there are 6649 targets files, which likely to be leakage, because there were only 150 pods on this pod. Unfortunately, <code class="language-plaintext highlighter-rouge">debugfs</code> can’t find any target files, so the output showed as meaningless string <code class="language-plaintext highlighter-rouge">"Inode Pathname"</code>.</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>kubelet pid=8412
+...
+Count: 7491
+...
+1	1	6649	Inode Pathname
+</code></pre></div></div>
+
+<h3 id="bad-apple">Bad apple</h3>
+<p>Why <code class="language-plaintext highlighter-rouge">debugfs</code> can’t help anymore? The reason is simple, each cgroup for a pod is using its own rootfs. This means the watcher are somehow residing on different rootfs and using independent inode index.  There are some other ways to do it, I choose the most common tool <code class="language-plaintext highlighter-rouge">grep</code> to find out.</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>sudo grep / -inum 1
+/home/kubernetes/containerized_mounter/rootfs/dev
+/home/kubernetes/containerized_mounter/rootfs/proc
+...
+/home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet/pods/5325873d-f2a0-48df-83e2-0b911df2f77f/volumes/kubernetes.io~projected/kube-api-access-227jg
+...
+/dev
+/boot/efi
+...
+</code></pre></div></div>
+
+<p>This turns things easy, because I can just use pod ID to compare between running pods to find out if there are any terminated pods are there. And it did show there’s some non-existence pods still being watched somehow.</p>
+
+<h2 id="what-to-expect-next">What to expect next</h2>
+
+<ul>
+  <li><a href="">How Kubelet leaked inotify watchers?</a></li>
+  <li><a href="">debugfs</a></li>
+</ul>
+
+<h2 id="references">References</h2>
+
+<div class="footnotes" role="doc-endnotes">
+  <ol>
+    <li id="fn:flbit_ino" role="doc-endnote">
+      <p><a href="https://stackoverflow.com/a/76712244">Fluentbit error “cannot adjust chunk size” on GKE</a> <a href="#fnref:flbit_ino" class="reversefootnote" role="doc-backlink">&#8617;</a></p>
+    </li>
+    <li id="fn:man_proc" role="doc-endnote">
+      <p><a href="https://manpages.courier-mta.org/htmlman5/proc.5.html">proc(5)</a> <a href="#fnref:man_proc" class="reversefootnote" role="doc-backlink">&#8617;</a></p>
+    </li>
+  </ol>
+</div>]]></content><author><name>Pengzhan Hao</name></author><category term="Kubernetes" /><category term="Kubelet" /><category term="Debug" /><category term="Inotify" /><summary type="html"><![CDATA[Symptom Recently, I faced an issue where Kubelet on a node reported error message failed to create file descriptors. error creating file watcher: too many open files error creating file watcher: no space left on device After short checking, I found the node has max_user_watches of 10000, but the TotalinotifyWatches is beyond this value. (P. S still not sure why watcher can initiate more than cap). In order to find which process occupied the most watchers. I used following command1 to find it out. echo -e "COUNT\tPID\tUSER\tCOMMAND"; sudo find /proc/[0-9]*/fdinfo -type f 2&gt;/dev/null | sudo xargs grep ^inotify 2&gt;/dev/null | cut -d/ -f 3 | uniq -c | sort -nr | { while read -rs COUNT PID; do echo -en "$COUNT\t$PID\t"; ps -p $PID -o user=,command=; done} COUNT PID USER COMMAND 7491 8412 root /home/kubernetes/bin/kubelet --v=2 --cloud-provide=gce --experi 2620 1 root /sbin/init .... Surprisingly, Kubelet initiated more than 7000 inotify watchers. I assumed there was an inotify leakage in Kubelet. Leakage check Clean Kubelet To better understand the situation, I created a clean cluster with only 1 clean node on GKE. Roughly 70 inotify watchers were there. I created a single nginx pod and the number increased by 3. Theoretically, these 3 watchers are used by Kubelet to monitor any changes on rootfs , kube-api-access and PodSandbox. But to verify it, we need to check more details on which inodes are monitored by Kubelet. Check inotify file descriptors To do so, let’s take a look how to track a single inotify file descriptor. Opened processes’ fdinfo folder, we can examine each or them to find an inotify fd. # Find kubelet pid ps -aux | grep kubelet KPID=2430 # File the an example fd sudo ls /proc/2430/fdinfo 0 1 10 11 12 13 14 2 3 4 5 6 7 8 9 ... sudo cat /proc/2430/fdinfo/8 pos: 0 flags: 02004000 mnt_id: 15 ino: 1057 inotify wd:1 ino:3f327 sdev:800001 mask:fc6 ignored_mask:0 fhandle-bytes:8 fhandle-type:1 f_handle:27f30300e5059ea2 This is very confusing, so I rely on man proc2 to understand every piece of them. In given fd, the needed information to continue sit in the last line. It’s an inotify entry represents the 1 file or folder to be monitored. And the most useful data is ino:3f327 which represents the inode number of target file (in hexadecimal). And sdev:800001, which represents the ID of device where the inode sit on, and it’s also in hex. Using lsblk, I can see there’s only 1 disk I’m using on the node, so finding the target file would be easy. # Cast to decimal ino=3f327 dec="$((16#${ino}))" # Find the target file loc="debugfs -R 'ncheck ${dec}' /dev/sda1" sudo eval $loc 2&gt;/dev/null debugfd 1.46.5 (30-Dec-2021) Inode Pathname 258855 /etc/srv/kubernetes/pki/ca-certificates.crt Put all processes above into one single script, I can retrieve all target files, that would help to understand if there’s a real leakage. Also, I count the unique inode amount, this could also help to know which inode are monitored multiple times. cat &lt;&lt; EOF | sudo tee -a test.sh echo "kubelet pid="${PID} in_fds=$(find /proc/${PID}/fdinfo -type f 2&gt;/dev/null | xargs grep ^inotify | cut -d " " -f 3 | cut -d ":" -f 2) echo ${in_fds} echo "Count: $(find /proc/${PID}/fdinfo -type f 2&gt;/dev/null | xargs grep ^inotify | wc -l)" uniq_fds=$(echo "${in_fds[@]}" | sort | uniq) echo ${uniq_fds} while read -r element; do count=$(echo "${in_fds[@]}" | grep -o "${element}" | wc -l) dec="$((16#${element}))" loc="debugfs -R 'ncheck ${dec}' /dev/sda1" loc=$(eval $loc 2&gt;/dev/null | tail -1 | cut -d " " -f 4) printf "%-6s %-10s %-6s %s\n" "${element}" "${dec}" "${count}" "${loc}" done &lt;&lt;&lt; "${uniq_fds}" EOF sudo bash test.sh kubelet pid=2430 3f327 3f321 ... Count: 120 1 10b 1259 128a ... 1 1 72 Inode Pathname 10b 267 1 267 /etc/systemd/system/multi-user.target.wants/snapd.service ... The given results are consists by following parts: One line for get Kubelet pid One line for all target inode numbers One line for tell how many unique inode (120) One line of sorted target Following 120 lines, each of them represents a unique inode number, its decimal number, count, another time of decimal number and the target file path. I used the same script to the problematic node, and it showed the following result. In summary, most Kubelet watchers were targeting ino:1 . And there are 6649 targets files, which likely to be leakage, because there were only 150 pods on this pod. Unfortunately, debugfs can’t find any target files, so the output showed as meaningless string "Inode Pathname". kubelet pid=8412 ... Count: 7491 ... 1 1 6649 Inode Pathname Bad apple Why debugfs can’t help anymore? The reason is simple, each cgroup for a pod is using its own rootfs. This means the watcher are somehow residing on different rootfs and using independent inode index. There are some other ways to do it, I choose the most common tool grep to find out. sudo grep / -inum 1 /home/kubernetes/containerized_mounter/rootfs/dev /home/kubernetes/containerized_mounter/rootfs/proc ... /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet/pods/5325873d-f2a0-48df-83e2-0b911df2f77f/volumes/kubernetes.io~projected/kube-api-access-227jg ... /dev /boot/efi ... This turns things easy, because I can just use pod ID to compare between running pods to find out if there are any terminated pods are there. And it did show there’s some non-existence pods still being watched somehow. What to expect next How Kubelet leaked inotify watchers? debugfs References Fluentbit error “cannot adjust chunk size” on GKE &#8617; proc(5) &#8617;]]></summary></entry><entry><title type="html">Debug Kubelet</title><link href="https://dev.pengzhan.dev/posts/Debug-kubelet" rel="alternate" type="text/html" title="Debug Kubelet" /><published>2024-04-10T03:34:00-04:00</published><updated>2024-04-10T23:51:19-04:00</updated><id>https://dev.pengzhan.dev/posts/Debug-kubelet</id><content type="html" xml:base="https://dev.pengzhan.dev/posts/Debug-kubelet"><![CDATA[<h2 id="debug-logs">Debug logs</h2>
 
 <p>Like all others program’s debugging, the most straightforward way for newbies and the easiest way for advanced developer is relying on logs. Same to debug <code class="language-plaintext highlighter-rouge">kubelet</code>, bumping up verbosity to show more logs is the most intuitive approach when facing an issue. Like most component in Kubernetes, <code class="language-plaintext highlighter-rouge">kubelet</code> uses <code class="language-plaintext highlighter-rouge">klog</code> for logging and there are 10 verbosity levels(0-9).</p>
 
@@ -65,7 +211,7 @@
 <p>By the time, this note was written. In <code class="language-plaintext highlighter-rouge">kubelet</code> related code, level 8 was only used in <code class="language-plaintext highlighter-rouge">pkg/kubelet/prober/prober_manager.go</code> and level 7 was only used in <code class="language-plaintext highlighter-rouge">pkg/kubelet/logs/container_log_manager.go</code>.  And there are  11 occurrences that level 6 was used, and all of them are not part of workload lifecycle related.</p>
 
 <h2 id="further-readings">Further readings</h2>
-<p>[Inotify watcher leaks in Kubelet]</p>]]></content><author><name>Pengzhan Hao</name></author><category term="Kubernetes" /><category term="Kubelet" /><category term="Debug" /><summary type="html"><![CDATA[Debug logs Like all others program’s debugging, the most straightforward way for newbies and the easiest way for advanced developer is relying on logs. Same to debug kubelet, bumping up verbosity to show more logs is the most intuitive approach when facing an issue. Like most component in Kubernetes, kubelet uses klog for logging and there are 10 verbosity levels(0-9). TL;DR: Bumping up to level 5 would satisfy most debugging needs. Level Meaning Example 0 Always on (Warning, Error, Fatal) https://github.com/kubernetes/kubernetes/blob/d9c54f69d4bb7ae1bb655e1a2a50297d615025b5/pkg/kubelet/kubelet.go#L757-L757 1 Default level logs when don’t want any verbosity https://github.com/kubernetes/kubernetes/blob/d9c54f69d4bb7ae1bb655e1a2a50297d615025b5/pkg/kubelet/kubelet.go#L2527 2 Most important logs when major operations happen, also the default verbosity level https://github.com/kubernetes/kubernetes/blob/d9c54f69d4bb7ae1bb655e1a2a50297d615025b5/pkg/kubelet/kubelet.go#L483-L483 3 Extended information https://github.com/kubernetes/kubernetes/blob/d9c54f69d4bb7ae1bb655e1a2a50297d615025b5/pkg/kubelet/kubelet.go#L2176 4 Debug level https://github.com/kubernetes/kubernetes/blob/d9c54f69d4bb7ae1bb655e1a2a50297d615025b5/pkg/kubelet/kubelet.go#L1731 5 Trace level https://github.com/kubernetes/kubernetes/blob/d9c54f69d4bb7ae1bb655e1a2a50297d615025b5/pkg/kubelet/kubelet.go#L2821-L2821 6 Display requested resources https://github.com/kubernetes/kubernetes/blob/d9c54f69d4bb7ae1bb655e1a2a50297d615025b5/pkg/kubelet/cm/cgroup_manager_linux.go#L401 7 Display HTTP request headers https://github.com/kubernetes/kubernetes/blob/d9c54f69d4bb7ae1bb655e1a2a50297d615025b5/pkg/kubelet/logs/container_log_manager.go#L299 8 Display HTTP request payload https://github.com/kubernetes/kubernetes/blob/d9c54f69d4bb7ae1bb655e1a2a50297d615025b5/pkg/kubelet/prober/prober_manager.go#L192 By the time, this note was written. In kubelet related code, level 8 was only used in pkg/kubelet/prober/prober_manager.go and level 7 was only used in pkg/kubelet/logs/container_log_manager.go. And there are 11 occurrences that level 6 was used, and all of them are not part of workload lifecycle related. Further readings [Inotify watcher leaks in Kubelet]]]></summary></entry><entry><title type="html">Labs of CS350</title><link href="https://blog.pengzhan.dev/posts/cs350-labs" rel="alternate" type="text/html" title="Labs of CS350" /><published>2022-02-22T16:08:17-05:00</published><updated>2022-05-04T19:45:56-04:00</updated><id>https://blog.pengzhan.dev/posts/cs350-labs</id><content type="html" xml:base="https://blog.pengzhan.dev/posts/cs350-labs"><![CDATA[<p>This will be a series regarding lab I gave during the spring 2022 semester.</p>
+<p><a href="/posts/inotify-watcher-leaks-in-kubelet.html">Inotify watcher leaks in Kubelet</a></p>]]></content><author><name>Pengzhan Hao</name></author><category term="Kubernetes" /><category term="Kubelet" /><category term="Debug" /><summary type="html"><![CDATA[Debug logs Like all others program’s debugging, the most straightforward way for newbies and the easiest way for advanced developer is relying on logs. Same to debug kubelet, bumping up verbosity to show more logs is the most intuitive approach when facing an issue. Like most component in Kubernetes, kubelet uses klog for logging and there are 10 verbosity levels(0-9). TL;DR: Bumping up to level 5 would satisfy most debugging needs. Level Meaning Example 0 Always on (Warning, Error, Fatal) https://github.com/kubernetes/kubernetes/blob/d9c54f69d4bb7ae1bb655e1a2a50297d615025b5/pkg/kubelet/kubelet.go#L757-L757 1 Default level logs when don’t want any verbosity https://github.com/kubernetes/kubernetes/blob/d9c54f69d4bb7ae1bb655e1a2a50297d615025b5/pkg/kubelet/kubelet.go#L2527 2 Most important logs when major operations happen, also the default verbosity level https://github.com/kubernetes/kubernetes/blob/d9c54f69d4bb7ae1bb655e1a2a50297d615025b5/pkg/kubelet/kubelet.go#L483-L483 3 Extended information https://github.com/kubernetes/kubernetes/blob/d9c54f69d4bb7ae1bb655e1a2a50297d615025b5/pkg/kubelet/kubelet.go#L2176 4 Debug level https://github.com/kubernetes/kubernetes/blob/d9c54f69d4bb7ae1bb655e1a2a50297d615025b5/pkg/kubelet/kubelet.go#L1731 5 Trace level https://github.com/kubernetes/kubernetes/blob/d9c54f69d4bb7ae1bb655e1a2a50297d615025b5/pkg/kubelet/kubelet.go#L2821-L2821 6 Display requested resources https://github.com/kubernetes/kubernetes/blob/d9c54f69d4bb7ae1bb655e1a2a50297d615025b5/pkg/kubelet/cm/cgroup_manager_linux.go#L401 7 Display HTTP request headers https://github.com/kubernetes/kubernetes/blob/d9c54f69d4bb7ae1bb655e1a2a50297d615025b5/pkg/kubelet/logs/container_log_manager.go#L299 8 Display HTTP request payload https://github.com/kubernetes/kubernetes/blob/d9c54f69d4bb7ae1bb655e1a2a50297d615025b5/pkg/kubelet/prober/prober_manager.go#L192 By the time, this note was written. In kubelet related code, level 8 was only used in pkg/kubelet/prober/prober_manager.go and level 7 was only used in pkg/kubelet/logs/container_log_manager.go. And there are 11 occurrences that level 6 was used, and all of them are not part of workload lifecycle related. Further readings Inotify watcher leaks in Kubelet]]></summary></entry><entry><title type="html">Labs of CS350</title><link href="https://dev.pengzhan.dev/posts/cs350-labs" rel="alternate" type="text/html" title="Labs of CS350" /><published>2022-02-22T16:08:17-05:00</published><updated>2022-05-04T19:45:56-04:00</updated><id>https://dev.pengzhan.dev/posts/cs350-labs</id><content type="html" xml:base="https://dev.pengzhan.dev/posts/cs350-labs"><![CDATA[<p>This will be a series regarding lab I gave during the spring 2022 semester.</p>
 
 <p>The reason why I am writing this down is because it has been a week and no students ask for the solution of the last Lab.
 I realise that learning gap between students are huge, especially when a non-profit university is admitting more and more students.
@@ -421,7 +567,7 @@ It’s time to switch from the running scheduler to the selected process. Wait f
       <p><a href="https://sourceware.org/binutils/docs/binutils/objcopy.html">3 objcopy - binutils mannual</a> <a href="#fnref:objcopyman" class="reversefootnote" role="doc-backlink">&#8617;</a></p>
     </li>
   </ol>
-</div>]]></content><author><name>Pengzhan Hao</name></author><category term="Xv6" /><category term="Teaching" /><category term="Operating system" /><category term="Binghamton university" /><summary type="html"><![CDATA[This will be a series regarding lab I gave during the spring 2022 semester. The reason why I am writing this down is because it has been a week and no students ask for the solution of the last Lab. I realise that learning gap between students are huge, especially when a non-profit university is admitting more and more students. To help all students in understanding concepts of modern OS, I decided to write this post. It starts with the past lab content I have (as the skelton), and will be amended with extra materials I think it helps. Remember, it’s for helping in learning. DON’T COPY &amp; PASTE CODE! Index Lab1: Introduction of Makefile and Xv6. Lab3: System calls for process management. Lab4: Inter-processes communication. Lab6/7: CPU scheduling. Lab1-Introduction Lab3-Process Lab4-IPC Lab6-7-Scheduling First user process in xv6 Kernel works In xv6, as the same as conventional linux OS, the very first user-level process is init. Before init’s running, all the OS bootstraps happen in a highly privileged mode(kernel level). Xv6’s kernel has the entry point as the main function located in the file main.c. The main function invokes 17 functions to set up kernel page tables, interrupt handlers, I/O devices and etc. When all kernel preparations are done, by calling the function userinit(), kernel will boot up process init. int main(void) { kinit1(end, P2V(4*1024*1024)); // phys page allocator kvmalloc(); // kernel page table mpinit(); // collect info about this machine lapicinit(); seginit(); // set up segments cprintf("\ncpu%d: starting xv6\n\n", cpu-&gt;id); picinit(); // interrupt controller ioapicinit(); // another interrupt controller consoleinit(); // I/O devices &amp; their interrupts uartinit(); // serial port pinit(); // process table tvinit(); // trap vectors binit(); // buffer cache fileinit(); // file table ideinit(); // disk if(!ismp) timerinit(); // uniprocessor timer startothers(); // start other processors kinit2(P2V(4*1024*1024), P2V(PHYSTOP)); // must come after startothers() userinit(); // first user process // Finish setting up this processor in mpmain. mpmain(); } It’s tricky since that init is a user process, but kernel can’t call any user-level system calls to create it. Why? 1. Kernel has all privileges to create a user process. So it doesn’t need to call system calls such as fork(). And 2. All other user processes can be created by forking from its parent. Forking including clone the whole user virtual memory layout. However, the first process has no parent to fork from. That’s why it makes the creation of the first user process becomes so unique. In proc.c, userinit() define there gives us the whole procedure of creating init. Similar to the fork(), but more simple. Process control block(structures for storing the process status) was created at the very first by calling allocproc(). After then, by invoking setupkvm()(defined in vm.c), kernel memory map was setup for the process. During setting up kernel memory map, a page size virtual memory will be assigned to the process as ready. And later, this page size memory will be used to store instructions of init. Followed by setup kernel stack for the init process, calling inituvm() will load init’s text into the page that is just being allocated. inituvm() takes 3 arguments: a pointer to the process’s page directory (p-&gt;pgdir), a char-type pointer declared from external which point to init’s text segment(_binary_initcode_start), and a char-type pointer which points to an external integer as the size of the init’s text segment(_binary_initcode_size). Simply put, it will load instructions of init into the memory. So now, the problem becomes when and where did instructions for init have compiled into the kernel? void userinit(void) { struct proc *p; extern char _binary_initcode_start[], _binary_initcode_size[]; p = allocproc(); initproc = p; if((p-&gt;pgdir = setupkvm()) == 0) panic("userinit: out of memory?"); inituvm(p-&gt;pgdir, _binary_initcode_start, (int)_binary_initcode_size); p-&gt;sz = PGSIZE; memset(p-&gt;tf, 0, sizeof(*p-&gt;tf)); p-&gt;tf-&gt;cs = (SEG_UCODE &lt;&lt; 3) | DPL_USER; p-&gt;tf-&gt;ds = (SEG_UDATA &lt;&lt; 3) | DPL_USER; p-&gt;tf-&gt;es = p-&gt;tf-&gt;ds; p-&gt;tf-&gt;ss = p-&gt;tf-&gt;ds; p-&gt;tf-&gt;eflags = FL_IF; p-&gt;tf-&gt;esp = PGSIZE; p-&gt;tf-&gt;eip = 0; // beginning of initcode.S safestrcpy(p-&gt;name, "initcode", sizeof(p-&gt;name)); p-&gt;cwd = namei("/"); p-&gt;state = RUNNABLE; } Where the user-level code was integrated? If you search the keyword “_binary_initcode_start” in the source code, you can’t find any references. The clue comes from the Makefile. In the makefile, initcode is a prerequisites to compile the kernel image. Step 1: Before kernel was compiled, initcode.S was first compiled to a runnable binary initcode. This binary was very odd because it was not supposed to let any other OS to run it. Initcode.s was first compiled without any standard including, and generating the intermediate file initcode.o. Step 2: Initcode.o then linked to Initcode.out with two uncommon settings. First it specify the entry of this binary file as when “start” symbol points to. This “start” symbol was declared in the assembly code. Second it specify a absolute address(0) for the text segments. By doing this, text segments will be placed at the start of the binary file (except the header of the ELF)1. Step 3: Initcode.out is already a minimized binary but it’s not enough. That’s why when using objcopy to copy it to the file initcode, it further strip all headers and debug information2. At this point, we have a minimal binary file initcode. From the first byte of this file, it’s only includes runnable instructions. And the size of the file is only 44 bytes. initcode: initcode.S $(CC) $(CFLAGS) -nostdinc -I. -c initcode.S # Step 1 $(LD) $(LDFLAGS) -N -e start -Ttext 0 -o initcode.out initcode.o # Step 2 $(OBJCOPY) -S -O binary initcode.out initcode # Step 3 $(OBJDUMP) -S initcode.o &gt; initcode.asm This binary later were appended to the kernel using following commands. And during this appending, 3 symbols were generated and added to the symbol table of the kernel1. “_binary_initcode_start” contains the address of where the initcode segment was appended to. “_binary_initcode_end” contains the address of where the initcode segment was ended at. “_binary_initcode_size” is a *ABS* type symbol with value 0x2C(45) that specify the size of the initcode segment is 45 bytes. kernel: $(OBJS) entry.o entryother initcode kernel.ld $(LD) $(LDFLAGS) -T kernel.ld -o kernel entry.o $(OBJS) -b binary initcode entryother # &lt;- This Line $(OBJDUMP) -S kernel &gt; kernel.asm $(OBJDUMP) -t kernel | sed '1,/SYMBOL TABLE/d; s/ .* / /; /^$$/d' &gt; kernel.sym In short summary, using objdump, we can verify that source code initcode.S has been compiled and loaded into the kernel. Also the segment of initcode’s instructions was located by the pointer “_binary_initcode_start”. That’s explain when calling inituvm(p-&gt;pgdir, _binary_initcode_start, (int)_binary_initcode_size);, functionalities implemented in initcode.S will be loaded into the runtime of the first process within xv6. # Header of the file kernel kernel: file format elf32-i386 kernel architecture: i386, flags 0x00000112: EXEC_P, HAS_SYMS, D_PAGED start address 0x0010000c Program Header: LOAD off 0x00001000 vaddr 0x80100000 paddr 0x00100000 align 2**12 filesz 0x00008c6a memsz 0x00008c6a flags r-x ... Sections: Idx Name Size VMA LMA File off Algn 0 .text 00008586 80100000 00100000 00001000 2**2 CONTENTS, ALLOC, LOAD, READONLY, CODE ... SYMBOL TABLE: ... 8010b50c g .data 00000000 _binary_initcode_end ... 8010b4e0 g .data 00000000 _binary_initcode_start ... 0000002c g *ABS* 00000000 _binary_initcode_size ... User-level code Take a look of content in the initcode.S, you will find the code can explain itself well. There are no other jobs but just calling system call exec to run a user-level binary “init”. Initcode.S: # Initial process execs /init. #include "syscall.h" #include "traps.h" # exec(init, argv) .globl start start: pushl $argv pushl $init pushl $0 // where caller pc would be movl $SYS_exec, %eax int $T_SYSCALL # for(;;) exit(); exit: movl $SYS_exit, %eax int $T_SYSCALL jmp exit # char init[] = "/init\0"; init: .string "/init\0" # char *argv[] = { init, 0 }; .p2align 2 argv: .long init .long 0 The “init” mentioned above is not a pure user-level binary executable that compiled from the source code init.c. Within init.c, a file named console will be created at the runtime for saving standard outputs and errors. Then it will forked a child process(the second user process), and let it run program “sh”. “sh” is the xv6’s default shell, a user-level program that generated from source sh.c. After the shell boots up, you can interactive with the xv6. This’s how first process (and second process) was started in the xv6. init.c: // init: The initial user-level program #include "types.h" #include "stat.h" #include "user.h" #include "fcntl.h" char *argv[] = { "sh", 0 }; int main(void) { int pid, wpid; if(open("console", O_RDWR) &lt; 0){ mknod("console", 1, 1); open("console", O_RDWR); } dup(0); // stdout dup(0); // stderr for(;;){ printf(1, "init: starting sh\n"); pid = fork(); if(pid &lt; 0){ printf(1, "init: fork failed\n"); exit(); } if(pid == 0){ exec("sh", argv); printf(1, "init: exec sh failed\n"); exit(); } while((wpid=wait()) &gt;= 0 &amp;&amp; wpid != pid) printf(1, "zombie!\n"); } } Xv6’s round robin schduler The Scheduler is the core of an operating system. With the scheduling of processes, the kernel can achieve near-real-time execution of multiple workloads. The scheduling problem is also an active aspect of computer science research. You can’t have one algorithm to fit all scenarios. Xv6 by default has a round-robin scheduler. It’s controlled using two-level for-loops, where the top-level for-loop is an endless loop that will keep the scheduler busy running. The second-level nested for-loop will iterate a data structure named Ptable where all control information for processes is stored. Information including pid, process name, etc. is stored in a structure called proc. Ptable is an array of processes. Every runnable process in the Ptable will run strictly 1 time tick until the for-loop reached the last process in the Ptable. Then it will loop back to the top-level for-loop for the next iteration of processes. // In file proc.c struct { struct spinlock lock; struct proc proc[NPROC]; } ptable; // In file proc.h struct proc { uint sz; // Size of process memory (bytes) pde_t* pgdir; // Page table char *kstack; // Bottom of kernel stack for this process enum procstate state; // Process state int pid; // Process ID struct proc *parent; // Parent process struct trapframe *tf; // Trap frame for current syscall struct context *context; // swtch() here to run process void *chan; // If non-zero, sleeping on chan int killed; // If non-zero, have been killed struct file *ofile[NOFILE]; // Open files struct inode *cwd; // Current directory char name[16]; // Process name (debugging) }; // In file proc.c void scheduler(void) { struct proc *p; for(;;){ // Enable interrupts on this processor. sti(); // Loop over process table looking for process to run. acquire(&amp;ptable.lock); for(p = ptable.proc; p &lt; &amp;ptable.proc[NPROC]; p++){ if(p-&gt;state != RUNNABLE) continue; // Switch to chosen process. It is the process's job // to release ptable.lock and then reacquire it // before jumping back to us. proc = p; switchuvm(p); p-&gt;state = RUNNING; swtch(&amp;cpu-&gt;scheduler, proc-&gt;context); switchkvm(); // Process is done running for now. // It should have changed its p-&gt;state before coming back. proc = 0; } release(&amp;ptable.lock); } } It’s not hard to understand why this logic makes a round-robin manner. This is very important to understand how to pick a process to run because scheduling is about always picking the appropriate process to achieve higher performance. You can always come up with some new ideas for designing a good scheduler policy. Understanding how to switch from one process to another is equivalently important. Once the process for the next time tick is selected. It’s time to switch from the running scheduler to the selected process. Wait for a second, there are two questions we haven’t answered. What is the running scheduler? How did the last running process stop running and give the CPU back to the scheduler? Lab ld(1) - Linux man page &#8617; &#8617;2 3 objcopy - binutils mannual &#8617;]]></summary></entry><entry><title type="html">EDDL: How do we train neural networks on limited edge devices - PART 2</title><link href="https://blog.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices-part2" rel="alternate" type="text/html" title="EDDL: How do we train neural networks on limited edge devices - PART 2" /><published>2021-10-31T13:01:14-04:00</published><updated>2021-12-11T13:35:39-05:00</updated><id>https://blog.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices-part2</id><content type="html" xml:base="https://blog.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices-part2"><![CDATA[<p>In the last post, part1, our idea of distributed learning on edge environment was generally addressed.
+</div>]]></content><author><name>Pengzhan Hao</name></author><category term="Xv6" /><category term="Teaching" /><category term="Operating system" /><category term="Binghamton university" /><summary type="html"><![CDATA[This will be a series regarding lab I gave during the spring 2022 semester. The reason why I am writing this down is because it has been a week and no students ask for the solution of the last Lab. I realise that learning gap between students are huge, especially when a non-profit university is admitting more and more students. To help all students in understanding concepts of modern OS, I decided to write this post. It starts with the past lab content I have (as the skelton), and will be amended with extra materials I think it helps. Remember, it’s for helping in learning. DON’T COPY &amp; PASTE CODE! Index Lab1: Introduction of Makefile and Xv6. Lab3: System calls for process management. Lab4: Inter-processes communication. Lab6/7: CPU scheduling. Lab1-Introduction Lab3-Process Lab4-IPC Lab6-7-Scheduling First user process in xv6 Kernel works In xv6, as the same as conventional linux OS, the very first user-level process is init. Before init’s running, all the OS bootstraps happen in a highly privileged mode(kernel level). Xv6’s kernel has the entry point as the main function located in the file main.c. The main function invokes 17 functions to set up kernel page tables, interrupt handlers, I/O devices and etc. When all kernel preparations are done, by calling the function userinit(), kernel will boot up process init. int main(void) { kinit1(end, P2V(4*1024*1024)); // phys page allocator kvmalloc(); // kernel page table mpinit(); // collect info about this machine lapicinit(); seginit(); // set up segments cprintf("\ncpu%d: starting xv6\n\n", cpu-&gt;id); picinit(); // interrupt controller ioapicinit(); // another interrupt controller consoleinit(); // I/O devices &amp; their interrupts uartinit(); // serial port pinit(); // process table tvinit(); // trap vectors binit(); // buffer cache fileinit(); // file table ideinit(); // disk if(!ismp) timerinit(); // uniprocessor timer startothers(); // start other processors kinit2(P2V(4*1024*1024), P2V(PHYSTOP)); // must come after startothers() userinit(); // first user process // Finish setting up this processor in mpmain. mpmain(); } It’s tricky since that init is a user process, but kernel can’t call any user-level system calls to create it. Why? 1. Kernel has all privileges to create a user process. So it doesn’t need to call system calls such as fork(). And 2. All other user processes can be created by forking from its parent. Forking including clone the whole user virtual memory layout. However, the first process has no parent to fork from. That’s why it makes the creation of the first user process becomes so unique. In proc.c, userinit() define there gives us the whole procedure of creating init. Similar to the fork(), but more simple. Process control block(structures for storing the process status) was created at the very first by calling allocproc(). After then, by invoking setupkvm()(defined in vm.c), kernel memory map was setup for the process. During setting up kernel memory map, a page size virtual memory will be assigned to the process as ready. And later, this page size memory will be used to store instructions of init. Followed by setup kernel stack for the init process, calling inituvm() will load init’s text into the page that is just being allocated. inituvm() takes 3 arguments: a pointer to the process’s page directory (p-&gt;pgdir), a char-type pointer declared from external which point to init’s text segment(_binary_initcode_start), and a char-type pointer which points to an external integer as the size of the init’s text segment(_binary_initcode_size). Simply put, it will load instructions of init into the memory. So now, the problem becomes when and where did instructions for init have compiled into the kernel? void userinit(void) { struct proc *p; extern char _binary_initcode_start[], _binary_initcode_size[]; p = allocproc(); initproc = p; if((p-&gt;pgdir = setupkvm()) == 0) panic("userinit: out of memory?"); inituvm(p-&gt;pgdir, _binary_initcode_start, (int)_binary_initcode_size); p-&gt;sz = PGSIZE; memset(p-&gt;tf, 0, sizeof(*p-&gt;tf)); p-&gt;tf-&gt;cs = (SEG_UCODE &lt;&lt; 3) | DPL_USER; p-&gt;tf-&gt;ds = (SEG_UDATA &lt;&lt; 3) | DPL_USER; p-&gt;tf-&gt;es = p-&gt;tf-&gt;ds; p-&gt;tf-&gt;ss = p-&gt;tf-&gt;ds; p-&gt;tf-&gt;eflags = FL_IF; p-&gt;tf-&gt;esp = PGSIZE; p-&gt;tf-&gt;eip = 0; // beginning of initcode.S safestrcpy(p-&gt;name, "initcode", sizeof(p-&gt;name)); p-&gt;cwd = namei("/"); p-&gt;state = RUNNABLE; } Where the user-level code was integrated? If you search the keyword “_binary_initcode_start” in the source code, you can’t find any references. The clue comes from the Makefile. In the makefile, initcode is a prerequisites to compile the kernel image. Step 1: Before kernel was compiled, initcode.S was first compiled to a runnable binary initcode. This binary was very odd because it was not supposed to let any other OS to run it. Initcode.s was first compiled without any standard including, and generating the intermediate file initcode.o. Step 2: Initcode.o then linked to Initcode.out with two uncommon settings. First it specify the entry of this binary file as when “start” symbol points to. This “start” symbol was declared in the assembly code. Second it specify a absolute address(0) for the text segments. By doing this, text segments will be placed at the start of the binary file (except the header of the ELF)1. Step 3: Initcode.out is already a minimized binary but it’s not enough. That’s why when using objcopy to copy it to the file initcode, it further strip all headers and debug information2. At this point, we have a minimal binary file initcode. From the first byte of this file, it’s only includes runnable instructions. And the size of the file is only 44 bytes. initcode: initcode.S $(CC) $(CFLAGS) -nostdinc -I. -c initcode.S # Step 1 $(LD) $(LDFLAGS) -N -e start -Ttext 0 -o initcode.out initcode.o # Step 2 $(OBJCOPY) -S -O binary initcode.out initcode # Step 3 $(OBJDUMP) -S initcode.o &gt; initcode.asm This binary later were appended to the kernel using following commands. And during this appending, 3 symbols were generated and added to the symbol table of the kernel1. “_binary_initcode_start” contains the address of where the initcode segment was appended to. “_binary_initcode_end” contains the address of where the initcode segment was ended at. “_binary_initcode_size” is a *ABS* type symbol with value 0x2C(45) that specify the size of the initcode segment is 45 bytes. kernel: $(OBJS) entry.o entryother initcode kernel.ld $(LD) $(LDFLAGS) -T kernel.ld -o kernel entry.o $(OBJS) -b binary initcode entryother # &lt;- This Line $(OBJDUMP) -S kernel &gt; kernel.asm $(OBJDUMP) -t kernel | sed '1,/SYMBOL TABLE/d; s/ .* / /; /^$$/d' &gt; kernel.sym In short summary, using objdump, we can verify that source code initcode.S has been compiled and loaded into the kernel. Also the segment of initcode’s instructions was located by the pointer “_binary_initcode_start”. That’s explain when calling inituvm(p-&gt;pgdir, _binary_initcode_start, (int)_binary_initcode_size);, functionalities implemented in initcode.S will be loaded into the runtime of the first process within xv6. # Header of the file kernel kernel: file format elf32-i386 kernel architecture: i386, flags 0x00000112: EXEC_P, HAS_SYMS, D_PAGED start address 0x0010000c Program Header: LOAD off 0x00001000 vaddr 0x80100000 paddr 0x00100000 align 2**12 filesz 0x00008c6a memsz 0x00008c6a flags r-x ... Sections: Idx Name Size VMA LMA File off Algn 0 .text 00008586 80100000 00100000 00001000 2**2 CONTENTS, ALLOC, LOAD, READONLY, CODE ... SYMBOL TABLE: ... 8010b50c g .data 00000000 _binary_initcode_end ... 8010b4e0 g .data 00000000 _binary_initcode_start ... 0000002c g *ABS* 00000000 _binary_initcode_size ... User-level code Take a look of content in the initcode.S, you will find the code can explain itself well. There are no other jobs but just calling system call exec to run a user-level binary “init”. Initcode.S: # Initial process execs /init. #include "syscall.h" #include "traps.h" # exec(init, argv) .globl start start: pushl $argv pushl $init pushl $0 // where caller pc would be movl $SYS_exec, %eax int $T_SYSCALL # for(;;) exit(); exit: movl $SYS_exit, %eax int $T_SYSCALL jmp exit # char init[] = "/init\0"; init: .string "/init\0" # char *argv[] = { init, 0 }; .p2align 2 argv: .long init .long 0 The “init” mentioned above is not a pure user-level binary executable that compiled from the source code init.c. Within init.c, a file named console will be created at the runtime for saving standard outputs and errors. Then it will forked a child process(the second user process), and let it run program “sh”. “sh” is the xv6’s default shell, a user-level program that generated from source sh.c. After the shell boots up, you can interactive with the xv6. This’s how first process (and second process) was started in the xv6. init.c: // init: The initial user-level program #include "types.h" #include "stat.h" #include "user.h" #include "fcntl.h" char *argv[] = { "sh", 0 }; int main(void) { int pid, wpid; if(open("console", O_RDWR) &lt; 0){ mknod("console", 1, 1); open("console", O_RDWR); } dup(0); // stdout dup(0); // stderr for(;;){ printf(1, "init: starting sh\n"); pid = fork(); if(pid &lt; 0){ printf(1, "init: fork failed\n"); exit(); } if(pid == 0){ exec("sh", argv); printf(1, "init: exec sh failed\n"); exit(); } while((wpid=wait()) &gt;= 0 &amp;&amp; wpid != pid) printf(1, "zombie!\n"); } } Xv6’s round robin schduler The Scheduler is the core of an operating system. With the scheduling of processes, the kernel can achieve near-real-time execution of multiple workloads. The scheduling problem is also an active aspect of computer science research. You can’t have one algorithm to fit all scenarios. Xv6 by default has a round-robin scheduler. It’s controlled using two-level for-loops, where the top-level for-loop is an endless loop that will keep the scheduler busy running. The second-level nested for-loop will iterate a data structure named Ptable where all control information for processes is stored. Information including pid, process name, etc. is stored in a structure called proc. Ptable is an array of processes. Every runnable process in the Ptable will run strictly 1 time tick until the for-loop reached the last process in the Ptable. Then it will loop back to the top-level for-loop for the next iteration of processes. // In file proc.c struct { struct spinlock lock; struct proc proc[NPROC]; } ptable; // In file proc.h struct proc { uint sz; // Size of process memory (bytes) pde_t* pgdir; // Page table char *kstack; // Bottom of kernel stack for this process enum procstate state; // Process state int pid; // Process ID struct proc *parent; // Parent process struct trapframe *tf; // Trap frame for current syscall struct context *context; // swtch() here to run process void *chan; // If non-zero, sleeping on chan int killed; // If non-zero, have been killed struct file *ofile[NOFILE]; // Open files struct inode *cwd; // Current directory char name[16]; // Process name (debugging) }; // In file proc.c void scheduler(void) { struct proc *p; for(;;){ // Enable interrupts on this processor. sti(); // Loop over process table looking for process to run. acquire(&amp;ptable.lock); for(p = ptable.proc; p &lt; &amp;ptable.proc[NPROC]; p++){ if(p-&gt;state != RUNNABLE) continue; // Switch to chosen process. It is the process's job // to release ptable.lock and then reacquire it // before jumping back to us. proc = p; switchuvm(p); p-&gt;state = RUNNING; swtch(&amp;cpu-&gt;scheduler, proc-&gt;context); switchkvm(); // Process is done running for now. // It should have changed its p-&gt;state before coming back. proc = 0; } release(&amp;ptable.lock); } } It’s not hard to understand why this logic makes a round-robin manner. This is very important to understand how to pick a process to run because scheduling is about always picking the appropriate process to achieve higher performance. You can always come up with some new ideas for designing a good scheduler policy. Understanding how to switch from one process to another is equivalently important. Once the process for the next time tick is selected. It’s time to switch from the running scheduler to the selected process. Wait for a second, there are two questions we haven’t answered. What is the running scheduler? How did the last running process stop running and give the CPU back to the scheduler? Lab ld(1) - Linux man page &#8617; &#8617;2 3 objcopy - binutils mannual &#8617;]]></summary></entry><entry><title type="html">EDDL: How do we train neural networks on limited edge devices - PART 2</title><link href="https://dev.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices-part2" rel="alternate" type="text/html" title="EDDL: How do we train neural networks on limited edge devices - PART 2" /><published>2021-10-31T13:01:14-04:00</published><updated>2021-12-11T13:35:39-05:00</updated><id>https://dev.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices-part2</id><content type="html" xml:base="https://dev.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices-part2"><![CDATA[<p>In the last post, part1, our idea of distributed learning on edge environment was generally addressed.
 I introduced the reason why edge distributed learning is needed and what improvements it can achieve.
 In this post, I will talk about our motivation study and how our framework works.</p>
 
@@ -525,7 +671,7 @@ The other platform is the <a href="https://www.raspberrypi.com/products/raspberr
 
 <p>The operating system running on the above platforms is Ubuntu 18.04 with Linux kernel 4.14.
 We used <a href="http://dlib.net/">Dlib</a>, a C++ library that provides implementations for a wide range of machine learning algorithms.
-We chose the Dlib library because it is written in C/C++, and can be easily and natively used in embedded devices.</p>]]></content><author><name>Pengzhan Hao</name></author><category term="Research" /><category term="Edge computing" /><summary type="html"><![CDATA[In the last post, part1, our idea of distributed learning on edge environment was generally addressed. I introduced the reason why edge distributed learning is needed and what improvements it can achieve. In this post, I will talk about our motivation study and how our framework works. How does data support us training on edge? Before designing and implementing our framework, we first need confirmation that training on edge resource-limited devices is worthwhile. We were using a malware detection neural network to show why a small, customized neural network is better. We collected 32000+ mobile apps feature as global data. With these data records, we trained a multilayer perceptron called “PerNet” to determine whether a given feature belongs to a benign or malware app. We called this detection. As well, PerNet can also classify malware apps into different types of attacks. We called this classification. The global model can achieve 93% above recall rate and 96.93% above accuracy. With all these data, we selected two community app usage sub-dataset for local model generations. Large categories (Scenario 1) We chose the 5 largest categories of apps, including entertainment, tools, brain&amp;Puzzle, Lifestyle, and Education, as well as the 5 largest malware categories. All together, 12000+ apps were included in this sub-dataset, almost 50 to 50 between benign and malware. Campus-community categories (Scenario 2) We chose the 5 most downloaded categories from college students as benign groups, as well as a similar amount of 5 malware categories. To ensure that malware apps are included in 5 benign categories, we also considered synthesizing some other malware apps within categories of 5 most downloaded(benign) categories. With these two types of sub-dataset, we used the same PerNet to generate multiple local models. Under each scenarios experiment, we compared global and local models on the preserved test dataset. In all classification performances, local beat global in every scenario. In detection performances, local also share the same accuracy as global does. In summary, local models were trained on special occasions. Under the same circumstance, a global model can achieve no better accuracy than local models. The reason why local is better might be because of overfitting. I believe this issue also be considered in the machine learning communities that they brought transfer learning, a technique to optimize global models to special scenarios but performing more training to a global model once it’s shipped to local. Design and Implementation Overall design The basic EDDL distributed training setup consists of 3 parts. EDDL training cluster, a device cluster that consists of edge or mobile devices that are participating in training. EDDL manager, the initial driver program that works as collect training data, relay data to training devices and initial training clusters. Training data entry (TDE), a data storage for all training data. Dynamic training data distribution Existing distributed DNN training solutions usually statically partition training data among workers. It can be a problem when the training node joins and exits. We designed our framework that can dynamically distribute training data during learning. Before every training batch started, a batch of TDE will be sent to devices. In our experiments, we found that by applying this design, overall training time was shortened by doing. Especially in large amount devices cases, this optimization can be 50% less than statically divided. Scaling up cluster size Our framework was designed to have both sync and async parameter aggregation. Asynchronous aggregation can allow a high outcome of training batch but with a sacrifice or converge time. Synchronous aggregation allows a quick converge time in epochs, however can’t ensure performance when there’s a struggler worker. As showed in experiments, we chose sync as default because the converging time is dominant in overall training time. But, we also considered the possibilities of that async with more workers can achieve similar overall training time. We introduced a formula to determine whether adding more training nodes can help or not. Here we used bandwidth usage coefficient (BUC) as \[BUC = \dfrac{n}{T_{sync}}\] In this formula, \(n\) is the number of devices, and \(T_{sync}\) is the transmission time of parameters. With an increasing number of workers, n increase linearly but transmission time does not. When \(BUC\) increases, the cluster can speed up training time by adding workers. Otherwise, adding more workers won’t help with overall training time. Adaptive leader role splitting The idea of role splitting is simple that a device can work as a worker as well leader. The advantage of doing this is straightforward that we can transfer 1 less parameter and training time will be shortened. However, in our current settings, it can’t perform much better help since only 1 leader role is in a cluster. We can benefit from this in our future works. Overall architecture Details were given in the image. Prototype hardware and software EDDL was designed to be run on two single-board computer embedded platforms. One such platform is ODROID-XU4, which is equipped with a 2.1/1.4 GHz 32-bit ARM processor and 2GB memory. The other platform is the Raspberry Pi 3 Model B board, which comes with an ARM 1.2 GHz 64-bit quad-core processor and 1GB memory. The operating system running on the above platforms is Ubuntu 18.04 with Linux kernel 4.14. We used Dlib, a C++ library that provides implementations for a wide range of machine learning algorithms. We chose the Dlib library because it is written in C/C++, and can be easily and natively used in embedded devices.]]></summary></entry><entry><title type="html">EDDL: How do we train neural networks on limited edge devices - PART 1</title><link href="https://blog.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices" rel="alternate" type="text/html" title="EDDL: How do we train neural networks on limited edge devices - PART 1" /><published>2021-10-13T16:53:20-04:00</published><updated>2021-12-11T13:35:39-05:00</updated><id>https://blog.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices</id><content type="html" xml:base="https://blog.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices"><![CDATA[<p>This post introduces our previous milestone in project “Edge trainer”, as the paper “EDDL: A Distributed Deep Learning System for Resource-limited Edge Computing Environment.” was published.
+We chose the Dlib library because it is written in C/C++, and can be easily and natively used in embedded devices.</p>]]></content><author><name>Pengzhan Hao</name></author><category term="Research" /><category term="Edge computing" /><summary type="html"><![CDATA[In the last post, part1, our idea of distributed learning on edge environment was generally addressed. I introduced the reason why edge distributed learning is needed and what improvements it can achieve. In this post, I will talk about our motivation study and how our framework works. How does data support us training on edge? Before designing and implementing our framework, we first need confirmation that training on edge resource-limited devices is worthwhile. We were using a malware detection neural network to show why a small, customized neural network is better. We collected 32000+ mobile apps feature as global data. With these data records, we trained a multilayer perceptron called “PerNet” to determine whether a given feature belongs to a benign or malware app. We called this detection. As well, PerNet can also classify malware apps into different types of attacks. We called this classification. The global model can achieve 93% above recall rate and 96.93% above accuracy. With all these data, we selected two community app usage sub-dataset for local model generations. Large categories (Scenario 1) We chose the 5 largest categories of apps, including entertainment, tools, brain&amp;Puzzle, Lifestyle, and Education, as well as the 5 largest malware categories. All together, 12000+ apps were included in this sub-dataset, almost 50 to 50 between benign and malware. Campus-community categories (Scenario 2) We chose the 5 most downloaded categories from college students as benign groups, as well as a similar amount of 5 malware categories. To ensure that malware apps are included in 5 benign categories, we also considered synthesizing some other malware apps within categories of 5 most downloaded(benign) categories. With these two types of sub-dataset, we used the same PerNet to generate multiple local models. Under each scenarios experiment, we compared global and local models on the preserved test dataset. In all classification performances, local beat global in every scenario. In detection performances, local also share the same accuracy as global does. In summary, local models were trained on special occasions. Under the same circumstance, a global model can achieve no better accuracy than local models. The reason why local is better might be because of overfitting. I believe this issue also be considered in the machine learning communities that they brought transfer learning, a technique to optimize global models to special scenarios but performing more training to a global model once it’s shipped to local. Design and Implementation Overall design The basic EDDL distributed training setup consists of 3 parts. EDDL training cluster, a device cluster that consists of edge or mobile devices that are participating in training. EDDL manager, the initial driver program that works as collect training data, relay data to training devices and initial training clusters. Training data entry (TDE), a data storage for all training data. Dynamic training data distribution Existing distributed DNN training solutions usually statically partition training data among workers. It can be a problem when the training node joins and exits. We designed our framework that can dynamically distribute training data during learning. Before every training batch started, a batch of TDE will be sent to devices. In our experiments, we found that by applying this design, overall training time was shortened by doing. Especially in large amount devices cases, this optimization can be 50% less than statically divided. Scaling up cluster size Our framework was designed to have both sync and async parameter aggregation. Asynchronous aggregation can allow a high outcome of training batch but with a sacrifice or converge time. Synchronous aggregation allows a quick converge time in epochs, however can’t ensure performance when there’s a struggler worker. As showed in experiments, we chose sync as default because the converging time is dominant in overall training time. But, we also considered the possibilities of that async with more workers can achieve similar overall training time. We introduced a formula to determine whether adding more training nodes can help or not. Here we used bandwidth usage coefficient (BUC) as \[BUC = \dfrac{n}{T_{sync}}\] In this formula, \(n\) is the number of devices, and \(T_{sync}\) is the transmission time of parameters. With an increasing number of workers, n increase linearly but transmission time does not. When \(BUC\) increases, the cluster can speed up training time by adding workers. Otherwise, adding more workers won’t help with overall training time. Adaptive leader role splitting The idea of role splitting is simple that a device can work as a worker as well leader. The advantage of doing this is straightforward that we can transfer 1 less parameter and training time will be shortened. However, in our current settings, it can’t perform much better help since only 1 leader role is in a cluster. We can benefit from this in our future works. Overall architecture Details were given in the image. Prototype hardware and software EDDL was designed to be run on two single-board computer embedded platforms. One such platform is ODROID-XU4, which is equipped with a 2.1/1.4 GHz 32-bit ARM processor and 2GB memory. The other platform is the Raspberry Pi 3 Model B board, which comes with an ARM 1.2 GHz 64-bit quad-core processor and 1GB memory. The operating system running on the above platforms is Ubuntu 18.04 with Linux kernel 4.14. We used Dlib, a C++ library that provides implementations for a wide range of machine learning algorithms. We chose the Dlib library because it is written in C/C++, and can be easily and natively used in embedded devices.]]></summary></entry><entry><title type="html">EDDL: How do we train neural networks on limited edge devices - PART 1</title><link href="https://dev.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices" rel="alternate" type="text/html" title="EDDL: How do we train neural networks on limited edge devices - PART 1" /><published>2021-10-13T16:53:20-04:00</published><updated>2021-12-11T13:35:39-05:00</updated><id>https://dev.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices</id><content type="html" xml:base="https://dev.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices"><![CDATA[<p>This post introduces our previous milestone in project “Edge trainer”, as the paper “EDDL: A Distributed Deep Learning System for Resource-limited Edge Computing Environment.” was published.
 As the first part of the introductions, I focus only on the motivation and summary of our works.
 More details in design and implementation can be found in late posts.</p>
 
@@ -602,7 +748,7 @@ Smartly schedule work balance and handle join/exit issues also need under consid
   Devices with sufficient bandwidth can also work as virtual leader devices.
   This approach helps minimize physical devices we used and more leaders can further scale up workers’ limits.</p>
   </li>
-</ul>]]></content><author><name>Pengzhan Hao</name></author><category term="Research" /><category term="Edge computing" /><summary type="html"><![CDATA[This post introduces our previous milestone in project “Edge trainer”, as the paper “EDDL: A Distributed Deep Learning System for Resource-limited Edge Computing Environment.” was published. As the first part of the introductions, I focus only on the motivation and summary of our works. More details in design and implementation can be found in late posts.]]></summary></entry><entry><title type="html">Generate Word Cloud Figures with Chinese-Tokenization and WordCloud python libraries</title><link href="https://blog.pengzhan.dev/posts/generate-word-cloud-with-chinese-fenci" rel="alternate" type="text/html" title="Generate Word Cloud Figures with Chinese-Tokenization and WordCloud python libraries" /><published>2020-09-15T22:00:14-04:00</published><updated>2021-12-11T13:35:39-05:00</updated><id>https://blog.pengzhan.dev/posts/generate-word-cloud-with-chinese-fenci</id><content type="html" xml:base="https://blog.pengzhan.dev/posts/generate-word-cloud-with-chinese-fenci"><![CDATA[<p>Let’s generate a word cloud like this. 
+</ul>]]></content><author><name>Pengzhan Hao</name></author><category term="Research" /><category term="Edge computing" /><summary type="html"><![CDATA[This post introduces our previous milestone in project “Edge trainer”, as the paper “EDDL: A Distributed Deep Learning System for Resource-limited Edge Computing Environment.” was published. As the first part of the introductions, I focus only on the motivation and summary of our works. More details in design and implementation can be found in late posts.]]></summary></entry><entry><title type="html">Generate Word Cloud Figures with Chinese-Tokenization and WordCloud python libraries</title><link href="https://dev.pengzhan.dev/posts/generate-word-cloud-with-chinese-fenci" rel="alternate" type="text/html" title="Generate Word Cloud Figures with Chinese-Tokenization and WordCloud python libraries" /><published>2020-09-15T22:00:14-04:00</published><updated>2021-12-11T13:35:39-05:00</updated><id>https://dev.pengzhan.dev/posts/generate-word-cloud-with-chinese-fenci</id><content type="html" xml:base="https://dev.pengzhan.dev/posts/generate-word-cloud-with-chinese-fenci"><![CDATA[<p>Let’s generate a word cloud like this. 
 Don’t understand the language is not a big deal.
 If your written language is based on latin alphabet(or other language has space between words), skip tokenization.</p>
 
@@ -761,7 +907,7 @@ If your written language is based on latin alphabet(or other language has space
 
 <p><img src="/static/2020-09/2020-06-28.png" height="150" /></p>
 
-<p>This generated word cloud figure reflects the most popular economy news’ keyword in the week started 06-28-2020. Two largest words in the figure are “新冠” and “新冠病毒”, both means “Covid-19” (This figure was in the week of the second covid spur in Beijing, China). The size of the image fits my phone screen and I can use an app to automatic sync it to my phone’s wallpaper. However, in this image, too many location nouns are presented. This will be something I can make progress on in the future.</p>]]></content><author><name>Pengzhan Hao</name></author><category term="Visualization" /><summary type="html"><![CDATA[Let’s generate a word cloud like this. Don’t understand the language is not a big deal. If your written language is based on latin alphabet(or other language has space between words), skip tokenization.]]></summary></entry><entry><title type="html">Xv6 introduction</title><link href="https://blog.pengzhan.dev/posts/intro-xv6" rel="alternate" type="text/html" title="Xv6 introduction" /><published>2017-07-28T14:56:55-04:00</published><updated>2021-12-11T13:35:39-05:00</updated><id>https://blog.pengzhan.dev/posts/intro-xv6</id><content type="html" xml:base="https://blog.pengzhan.dev/posts/intro-xv6"><![CDATA[<p>In this post, you will learn a few basic concepts of xv6. Learning path will be closed coupled to first project assignment I gave when I assisted in teaching OS classes.
+<p>This generated word cloud figure reflects the most popular economy news’ keyword in the week started 06-28-2020. Two largest words in the figure are “新冠” and “新冠病毒”, both means “Covid-19” (This figure was in the week of the second covid spur in Beijing, China). The size of the image fits my phone screen and I can use an app to automatic sync it to my phone’s wallpaper. However, in this image, too many location nouns are presented. This will be something I can make progress on in the future.</p>]]></content><author><name>Pengzhan Hao</name></author><category term="Visualization" /><summary type="html"><![CDATA[Let’s generate a word cloud like this. Don’t understand the language is not a big deal. If your written language is based on latin alphabet(or other language has space between words), skip tokenization.]]></summary></entry><entry><title type="html">Xv6 introduction</title><link href="https://dev.pengzhan.dev/posts/intro-xv6" rel="alternate" type="text/html" title="Xv6 introduction" /><published>2017-07-28T14:56:55-04:00</published><updated>2021-12-11T13:35:39-05:00</updated><id>https://dev.pengzhan.dev/posts/intro-xv6</id><content type="html" xml:base="https://dev.pengzhan.dev/posts/intro-xv6"><![CDATA[<p>In this post, you will learn a few basic concepts of xv6. Learning path will be closed coupled to first project assignment I gave when I assisted in teaching OS classes.
 Understand system call and know how to implement a simple one will be coved as the first half.
 In the second half of this post, I will discuss a little bit more on how to debug xv6 using gdb.<br />
 <!--more--></p>
@@ -836,7 +982,7 @@ Using ssh may connect to different physical devices under same domain name, this
 
 <div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>target remote localhost:28467
 <span class="c"># target remote [ip-addr]:28467</span>
-</code></pre></div></div>]]></content><author><name>Pengzhan Hao</name></author><category term="Xv6" /><category term="Teaching" /><category term="Operating system" /><summary type="html"><![CDATA[In this post, you will learn a few basic concepts of xv6. Learning path will be closed coupled to first project assignment I gave when I assisted in teaching OS classes. Understand system call and know how to implement a simple one will be coved as the first half. In the second half of this post, I will discuss a little bit more on how to debug xv6 using gdb.]]></summary></entry><entry><title type="html">Some of my previews experiment works: 2016</title><link href="https://blog.pengzhan.dev/posts/some-of-my-previews-exper-work" rel="alternate" type="text/html" title="Some of my previews experiment works: 2016" /><published>2016-10-28T12:27:33-04:00</published><updated>2021-12-11T13:35:39-05:00</updated><id>https://blog.pengzhan.dev/posts/some-of-my-previews-exper-work</id><content type="html" xml:base="https://blog.pengzhan.dev/posts/some-of-my-previews-exper-work"><![CDATA[<p>This blog contains only some basic record of my works. For some details, I will write a unique blog just for some specific topics.
+</code></pre></div></div>]]></content><author><name>Pengzhan Hao</name></author><category term="Xv6" /><category term="Teaching" /><category term="Operating system" /><summary type="html"><![CDATA[In this post, you will learn a few basic concepts of xv6. Learning path will be closed coupled to first project assignment I gave when I assisted in teaching OS classes. Understand system call and know how to implement a simple one will be coved as the first half. In the second half of this post, I will discuss a little bit more on how to debug xv6 using gdb.]]></summary></entry><entry><title type="html">Some of my previews experiment works: 2016</title><link href="https://dev.pengzhan.dev/posts/some-of-my-previews-exper-work" rel="alternate" type="text/html" title="Some of my previews experiment works: 2016" /><published>2016-10-28T12:27:33-04:00</published><updated>2021-12-11T13:35:39-05:00</updated><id>https://dev.pengzhan.dev/posts/some-of-my-previews-exper-work</id><content type="html" xml:base="https://dev.pengzhan.dev/posts/some-of-my-previews-exper-work"><![CDATA[<p>This blog contains only some basic record of my works. For some details, I will write a unique blog just for some specific topics.
 <!--more--></p>
 
 <h1 id="2016-10">2016-10</h1>
@@ -946,7 +1092,7 @@ unxz ubuntu-14.04lts-xubuntu-odroid-xu-20140714.img.xz
 <span class="c"># dump file</span>
 su
 <span class="nb">dd </span><span class="k">if</span><span class="o">=</span>/dev/block/mmcblk0p37 <span class="nv">of</span><span class="o">=</span>/sdcard/boot.img
-</code></pre></div></div>]]></content><author><name>Pengzhan Hao</name></author><category term="Research" /><category term="Log" /><category term="Miscellanies" /><summary type="html"><![CDATA[This blog contains only some basic record of my works. For some details, I will write a unique blog just for some specific topics.]]></summary></entry><entry><title type="html">Using charles proxy to monitor mobile SSL traffics</title><link href="https://blog.pengzhan.dev/posts/charles-is-not-a-good-tool" rel="alternate" type="text/html" title="Using charles proxy to monitor mobile SSL traffics" /><published>2016-10-27T22:50:33-04:00</published><updated>2021-12-11T13:35:39-05:00</updated><id>https://blog.pengzhan.dev/posts/charles-is-not-a-good-tool</id><content type="html" xml:base="https://blog.pengzhan.dev/posts/charles-is-not-a-good-tool"><![CDATA[<p>In this blog, I will generally talk about how to use proper tools to monitor SSL traffics of a mobile devices. Currently, I only can dealing with those SSL traffics which use an obviously certification. Some applications may not using system root cert or they doesn’t provide us a method to modify their own certs. For these situation, I still didn’t find a good solutions for it. But I’ll keep updating this if I get one.<br />
+</code></pre></div></div>]]></content><author><name>Pengzhan Hao</name></author><category term="Research" /><category term="Log" /><category term="Miscellanies" /><summary type="html"><![CDATA[This blog contains only some basic record of my works. For some details, I will write a unique blog just for some specific topics.]]></summary></entry><entry><title type="html">Using charles proxy to monitor mobile SSL traffics</title><link href="https://dev.pengzhan.dev/posts/charles-is-not-a-good-tool" rel="alternate" type="text/html" title="Using charles proxy to monitor mobile SSL traffics" /><published>2016-10-27T22:50:33-04:00</published><updated>2021-12-11T13:35:39-05:00</updated><id>https://dev.pengzhan.dev/posts/charles-is-not-a-good-tool</id><content type="html" xml:base="https://dev.pengzhan.dev/posts/charles-is-not-a-good-tool"><![CDATA[<p>In this blog, I will generally talk about how to use proper tools to monitor SSL traffics of a mobile devices. Currently, I only can dealing with those SSL traffics which use an obviously certification. Some applications may not using system root cert or they doesn’t provide us a method to modify their own certs. For these situation, I still didn’t find a good solutions for it. But I’ll keep updating this if I get one.<br />
 My current solution is using AP to forward all SSL traffic to a proxy, <a href="https://www.charlesproxy.com/">charles proxy</a> is my first choice (Prof asked). It’s a non-free software which still update new versions now. So mainly, I’ll talk about how to charles SSL proxy.
 <!--more--></p>
 
@@ -975,4 +1121,4 @@ You also need to save charles Root Certificate, it also contains in the same men
 
 <ul>
   <li>Set Proxy and SSL Proxy</li>
-</ul>]]></content><author><name>Pengzhan Hao</name></author><category term="Network" /><category term="Charles proxy" /><summary type="html"><![CDATA[In this blog, I will generally talk about how to use proper tools to monitor SSL traffics of a mobile devices. Currently, I only can dealing with those SSL traffics which use an obviously certification. Some applications may not using system root cert or they doesn’t provide us a method to modify their own certs. For these situation, I still didn’t find a good solutions for it. But I’ll keep updating this if I get one. My current solution is using AP to forward all SSL traffic to a proxy, charles proxy is my first choice (Prof asked). It’s a non-free software which still update new versions now. So mainly, I’ll talk about how to charles SSL proxy.]]></summary></entry><entry><title type="html">STSD: Stop Talking Start Doing</title><link href="https://blog.pengzhan.dev/posts/welcome-to-my-blog" rel="alternate" type="text/html" title="STSD: Stop Talking Start Doing" /><published>2016-10-26T22:50:33-04:00</published><updated>2024-04-03T18:23:31-04:00</updated><id>https://blog.pengzhan.dev/posts/welcome-to-my-blog</id><content type="html" xml:base="https://blog.pengzhan.dev/posts/welcome-to-my-blog"><![CDATA[]]></content><author><name>Pengzhan Hao</name><email>haopengzhan@gmail.com</email></author><category term="Diary" /><summary type="html"><![CDATA[]]></summary></entry></feed>
\ No newline at end of file
+</ul>]]></content><author><name>Pengzhan Hao</name></author><category term="Network" /><category term="Charles proxy" /><summary type="html"><![CDATA[In this blog, I will generally talk about how to use proper tools to monitor SSL traffics of a mobile devices. Currently, I only can dealing with those SSL traffics which use an obviously certification. Some applications may not using system root cert or they doesn’t provide us a method to modify their own certs. For these situation, I still didn’t find a good solutions for it. But I’ll keep updating this if I get one. My current solution is using AP to forward all SSL traffic to a proxy, charles proxy is my first choice (Prof asked). It’s a non-free software which still update new versions now. So mainly, I’ll talk about how to charles SSL proxy.]]></summary></entry><entry><title type="html">STSD: Stop Talking Start Doing</title><link href="https://dev.pengzhan.dev/posts/welcome-to-my-blog" rel="alternate" type="text/html" title="STSD: Stop Talking Start Doing" /><published>2016-10-26T22:50:33-04:00</published><updated>2024-04-03T18:23:31-04:00</updated><id>https://dev.pengzhan.dev/posts/welcome-to-my-blog</id><content type="html" xml:base="https://dev.pengzhan.dev/posts/welcome-to-my-blog"><![CDATA[]]></content><author><name>Pengzhan Hao</name><email>haopengzhan@gmail.com</email></author><category term="Diary" /><summary type="html"><![CDATA[]]></summary></entry></feed>
\ No newline at end of file
diff --git a/_site/index.html b/_site/index.html
index 2a20b7b..ca7d07b 100644
--- a/_site/index.html
+++ b/_site/index.html
@@ -5,7 +5,7 @@
 
 <meta name="description" content="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
 ">
-<link rel="canonical" href="https://blog.pengzhan.dev/"><link rel="alternate" type="application/rss+xml" title="STSD" href="/feed.xml"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
+<link rel="canonical" href="https://dev.pengzhan.dev/proxy/4000/"><link rel="alternate" type="application/rss+xml" title="STSD" href="/proxy/4000/feed.xml"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
 <link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png">
 <link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png">
 <link rel="manifest" href="/assets/site.webmanifest">
@@ -14,7 +14,7 @@
 <meta name="msapplication-TileColor" content="#da532c">
 <meta name="msapplication-config" content="/assets/browserconfig.xml">
 <meta name="theme-color" content="#ffffff">
-<link rel="stylesheet" href="/assets/css/main.css"><link rel="stylesheet" href="/static/2022-09/all.css" >
+<link rel="stylesheet" href="/proxy/4000/assets/css/main.css"><link rel="stylesheet" href="/static/2022-09/all.css" >
 
 <style>
     @import url('https://fonts.googleapis.com/css2?family=Poppins:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;0,800;0,900;1,100;1,200;1,300;1,400;1,500;1,600;1,700;1,800;1,900&family=Roboto+Slab:wght@100..900&family=ZCOOL+XiaoWei&display=swap')
@@ -27,15 +27,15 @@
 <meta property="og:locale" content="en" />
 <meta name="description" content="My personal blog, some contents are useful, the others are not. Just like my mediocre life." />
 <meta property="og:description" content="My personal blog, some contents are useful, the others are not. Just like my mediocre life." />
-<link rel="canonical" href="https://blog.pengzhan.dev/" />
-<meta property="og:url" content="https://blog.pengzhan.dev/" />
+<link rel="canonical" href="https://dev.pengzhan.dev/" />
+<meta property="og:url" content="https://dev.pengzhan.dev/" />
 <meta property="og:site_name" content="STSD" />
 <meta property="og:type" content="website" />
-<link rel="next" href="https://blog.pengzhan.dev/page2" />
+<link rel="next" href="https://dev.pengzhan.dev/page2" />
 <meta name="twitter:card" content="summary" />
 <meta property="twitter:title" content="STSD" />
 <script type="application/ld+json">
-{"@context":"https://schema.org","@type":"WebSite","author":{"@type":"Person","name":"Pengzhan Hao"},"dateModified":"2021-10-25T14:53:15-04:00","description":"My personal blog, some contents are useful, the others are not. Just like my mediocre life.","headline":"STSD","name":"STSD","url":"https://blog.pengzhan.dev/"}</script>
+{"@context":"https://schema.org","@type":"WebSite","author":{"@type":"Person","name":"Pengzhan Hao"},"dateModified":"2021-10-25T14:53:15-04:00","description":"My personal blog, some contents are useful, the others are not. Just like my mediocre life.","headline":"STSD","name":"STSD","url":"https://dev.pengzhan.dev/"}</script>
 <!-- End Jekyll SEO tag -->
 
 <!-- end custom head snippets -->
@@ -272,7 +272,7 @@
         }
       },
       paths: {
-        search_js: '/assets/search.js'
+        search_js: '/proxy/4000/assets/search.js'
       }
     };
     window.TEXT_VARIABLES = TEXT_VARIABLES;
@@ -427,9 +427,9 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
 </g>
 </svg>
 -->
-          <img src="/assets/favicon-32x32.png" style="margin-right: 5px;"><a title="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
-" href="/">STSD</a></div><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></div><nav class="navigation">
-        <ul><li class="navigation__item"><a href="/archive.html">Archive</a></li><li class="navigation__item"><a href="https://pengzhan.dev">About</a></li><li><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></li></ul>
+          <img src="/proxy/4000/assets/favicon-32x32.png" style="margin-right: 5px;"><a title="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
+" href="/proxy/4000/">STSD</a></div><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></div><nav class="navigation">
+        <ul><li class="navigation__item"><a href="/proxy/4000/archive.html">Archive</a></li><li class="navigation__item"><a href="https://pengzhan.dev">About</a></li><li><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></li></ul>
       </nav></div>
   </header>
 </div><div class="page__content"><div class ="main"><div class="grid grid--reverse">
@@ -439,134 +439,147 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
               <div class="col-main cell cell--auto"><!-- start custom main top snippet -->
 
 <!-- end custom main top snippet -->
-<article itemscope itemtype="http://schema.org/WebPage"><header style="display:none;"><h1>Home</h1></header><meta itemprop="headline" content="Home"><meta itemprop="author" content="Pengzhan Hao"/><div class="js-article-content"><div class="layout--articles"><div class="article-list items items--divided"><article class="item" itemscope itemtype="http://schema.org/BlogPosting"><div class="item__image" style="vertical-align: middle"><img class="image" src="/static/2024-04/Kubelet.webp" /></div><div class="item__content">
-          <header><a href="/posts/Debug-kubelet"><h2 itemprop="headline" class="item__header">Debug Kubelet</h2></a></header>
+<article itemscope itemtype="http://schema.org/WebPage"><header style="display:none;"><h1>Home</h1></header><meta itemprop="headline" content="Home"><meta itemprop="author" content="Pengzhan Hao"/><div class="js-article-content"><div class="layout--articles"><div class="article-list items items--divided"><article class="item" itemscope itemtype="http://schema.org/BlogPosting"><div class="item__image" style="vertical-align: middle"><img class="image" src="/proxy/4000/static/2024-04/kubelet_inotify_leak_logo.png" /></div><div class="item__content">
+          <header><a href="/proxy/4000/posts/inotify-watcher-leaks-in-kubelet"><h2 itemprop="headline" class="item__header">Inotify watcher leaks in Kubelet</h2></a></header>
+          <div class="item__description"><div class="article__content" itemprop="description articleBody">Symptom
+Recently, I faced an issue where Kubelet on a node reported error message failed to create file descriptors.
+
+error creating file watcher: too many open files
+error creating file watcher: no space left on device
+
+
+After short checking, I found the node has max_user_watches of 10000, but the TotalinotifyWatches is beyond this value. (P. S...</div><p><a href="/proxy/4000/posts/inotify-watcher-leaks-in-kubelet">Read more</a></p></div><div class="article__info clearfix"><ul class="left-col menu"><li>
+              <a class="button button--secondary button--pill button--sm"
+                href="/proxy/4000/archive.html?tag=Kubernetes">Kubernetes</a>
+            </li><li>
+              <a class="button button--secondary button--pill button--sm"
+                href="/proxy/4000/archive.html?tag=Kubelet">Kubelet</a>
+            </li><li>
+              <a class="button button--secondary button--pill button--sm"
+                href="/proxy/4000/archive.html?tag=Debug">Debug</a>
+            </li><li>
+              <a class="button button--secondary button--pill button--sm"
+                href="/proxy/4000/archive.html?tag=Inotify">Inotify</a>
+            </li></ul><ul class="right-col menu"><li><i class="fas fa-user"></i> <span>Pengzhan Hao</span></li><li><i class="far fa-calendar-alt"></i> <span>Apr 18, 2024</span>
+            </li></ul></div><meta itemprop="author" content="Pengzhan Hao"/><meta itemprop="datePublished" content="2024-04-18T16:35:00-04:00">
+    <meta itemprop="keywords" content="Kubernetes,Kubelet,Debug,Inotify"></div>
+      </article><article class="item" itemscope itemtype="http://schema.org/BlogPosting"><div class="item__image" style="vertical-align: middle"><img class="image" src="/proxy/4000/static/2024-04/Kubelet.webp" /></div><div class="item__content">
+          <header><a href="/proxy/4000/posts/Debug-kubelet"><h2 itemprop="headline" class="item__header">Debug Kubelet</h2></a></header>
           <div class="item__description"><div class="article__content" itemprop="description articleBody">Debug logs
 
-Like all others program’s debugging, the most straightforward way for newbies and the easiest way for advanced developer is relying on logs. Same to debug kubelet, bumping up verbosity to show more logs is the most intuitive approach when facing an issue. Like most component in Kubernetes, kubelet uses klog for logging and there are ...</div><p><a href="/posts/Debug-kubelet">Read more</a></p></div><div class="article__info clearfix"><ul class="left-col menu"><li>
+Like all others program’s debugging, the most straightforward way for newbies and the easiest way for advanced developer is relying on logs. Same to debug kubelet, bumping up verbosity to show more logs is the most intuitive approach when facing an issue. Like most component in Kubernetes, kubelet uses klog for logging and there are ...</div><p><a href="/proxy/4000/posts/Debug-kubelet">Read more</a></p></div><div class="article__info clearfix"><ul class="left-col menu"><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Kubernetes">Kubernetes</a>
+                href="/proxy/4000/archive.html?tag=Kubernetes">Kubernetes</a>
             </li><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Kubelet">Kubelet</a>
+                href="/proxy/4000/archive.html?tag=Kubelet">Kubelet</a>
             </li><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Debug">Debug</a>
+                href="/proxy/4000/archive.html?tag=Debug">Debug</a>
             </li></ul><ul class="right-col menu"><li><i class="fas fa-user"></i> <span>Pengzhan Hao</span></li><li><i class="far fa-calendar-alt"></i> <span>Apr 10, 2024</span>
             </li></ul></div><meta itemprop="author" content="Pengzhan Hao"/><meta itemprop="datePublished" content="2024-04-10T03:34:00-04:00">
     <meta itemprop="keywords" content="Kubernetes,Kubelet,Debug"></div>
-      </article><article class="item" itemscope itemtype="http://schema.org/BlogPosting"><div class="item__image" style="vertical-align: middle"><img class="image" src="/static/2022-02/BU.jpeg" /></div><div class="item__content">
-          <header><a href="/posts/cs350-labs"><h2 itemprop="headline" class="item__header">Labs of CS350</h2></a></header>
+      </article><article class="item" itemscope itemtype="http://schema.org/BlogPosting"><div class="item__image" style="vertical-align: middle"><img class="image" src="/proxy/4000/static/2022-02/BU.jpeg" /></div><div class="item__content">
+          <header><a href="/proxy/4000/posts/cs350-labs"><h2 itemprop="headline" class="item__header">Labs of CS350</h2></a></header>
           <div class="item__description"><div class="article__content" itemprop="description articleBody">This will be a series regarding lab I gave during the spring 2022 semester.
 
 The reason why I am writing this down is because it has been a week and no students ask for the solution of the last Lab.
 I realise that learning gap between students are huge, especially when a non-profit university is admitting more and more students.
-To help all stud...</div><p><a href="/posts/cs350-labs">Read more</a></p></div><div class="article__info clearfix"><ul class="left-col menu"><li>
+To help all stud...</div><p><a href="/proxy/4000/posts/cs350-labs">Read more</a></p></div><div class="article__info clearfix"><ul class="left-col menu"><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Xv6">Xv6</a>
+                href="/proxy/4000/archive.html?tag=Xv6">Xv6</a>
             </li><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Teaching">Teaching</a>
+                href="/proxy/4000/archive.html?tag=Teaching">Teaching</a>
             </li><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Operating+system">Operating system</a>
+                href="/proxy/4000/archive.html?tag=Operating+system">Operating system</a>
             </li><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Binghamton+university">Binghamton university</a>
+                href="/proxy/4000/archive.html?tag=Binghamton+university">Binghamton university</a>
             </li></ul><ul class="right-col menu"><li><i class="fas fa-user"></i> <span>Pengzhan Hao</span></li><li><i class="far fa-calendar-alt"></i> <span>Feb 22, 2022</span>
             </li></ul></div><meta itemprop="author" content="Pengzhan Hao"/><meta itemprop="datePublished" content="2022-02-22T16:08:17-05:00">
     <meta itemprop="keywords" content="Xv6,Teaching,Operating system,Binghamton university"></div>
-      </article><article class="item" itemscope itemtype="http://schema.org/BlogPosting"><div class="item__image" style="vertical-align: middle"><img class="image" src="/static/2021-10/f.5_Impl_leader_worker.png" /></div><div class="item__content">
-          <header><a href="/posts/eddl-how-do-we-train-on-limited-edge-devices-part2"><h2 itemprop="headline" class="item__header">EDDL: How do we train neural networks on limited edge devices - PART 2</h2></a></header>
+      </article><article class="item" itemscope itemtype="http://schema.org/BlogPosting"><div class="item__image" style="vertical-align: middle"><img class="image" src="/proxy/4000/static/2021-10/f.5_Impl_leader_worker.png" /></div><div class="item__content">
+          <header><a href="/proxy/4000/posts/eddl-how-do-we-train-on-limited-edge-devices-part2"><h2 itemprop="headline" class="item__header">EDDL: How do we train neural networks on limited edge devices - PART 2</h2></a></header>
           <div class="item__description"><div class="article__content" itemprop="description articleBody">In the last post, part1, our idea of distributed learning on edge environment was generally addressed.
 I introduced the reason why edge distributed learning is needed and what improvements it can achieve.
 In this post, I will talk about our motivation study and how our framework works.
 
 How does data support us training on edge?
 
-Before designin...</div><p><a href="/posts/eddl-how-do-we-train-on-limited-edge-devices-part2">Read more</a></p></div><div class="article__info clearfix"><ul class="left-col menu"><li>
+Before designin...</div><p><a href="/proxy/4000/posts/eddl-how-do-we-train-on-limited-edge-devices-part2">Read more</a></p></div><div class="article__info clearfix"><ul class="left-col menu"><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Research">Research</a>
+                href="/proxy/4000/archive.html?tag=Research">Research</a>
             </li><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Edge+computing">Edge computing</a>
+                href="/proxy/4000/archive.html?tag=Edge+computing">Edge computing</a>
             </li></ul><ul class="right-col menu"><li><i class="fas fa-user"></i> <span>Pengzhan Hao</span></li><li><i class="far fa-calendar-alt"></i> <span>Oct 31, 2021</span>
             </li></ul></div><meta itemprop="author" content="Pengzhan Hao"/><meta itemprop="datePublished" content="2021-10-31T13:01:14-04:00">
     <meta itemprop="keywords" content="Research,Edge computing"></div>
-      </article><article class="item" itemscope itemtype="http://schema.org/BlogPosting"><div class="item__image" style="vertical-align: middle"><img class="image" src="/static/2021-10/edgelearn-1.png" /></div><div class="item__content">
-          <header><a href="/posts/eddl-how-do-we-train-on-limited-edge-devices"><h2 itemprop="headline" class="item__header">EDDL: How do we train neural networks on limited edge devices - PART 1</h2></a></header>
+      </article><article class="item" itemscope itemtype="http://schema.org/BlogPosting"><div class="item__image" style="vertical-align: middle"><img class="image" src="/proxy/4000/static/2021-10/edgelearn-1.png" /></div><div class="item__content">
+          <header><a href="/proxy/4000/posts/eddl-how-do-we-train-on-limited-edge-devices"><h2 itemprop="headline" class="item__header">EDDL: How do we train neural networks on limited edge devices - PART 1</h2></a></header>
           <div class="item__description"><div class="article__content" itemprop="description articleBody">This post introduces our previous milestone in project “Edge trainer”, as the paper “EDDL: A Distributed Deep Learning System for Resource-limited Edge Computing Environment.” was published.
 As the first part of the introductions, I focus only on the motivation and summary of our works.
-More details in design and implementation can be found in l...</div><p><a href="/posts/eddl-how-do-we-train-on-limited-edge-devices">Read more</a></p></div><div class="article__info clearfix"><ul class="left-col menu"><li>
+More details in design and implementation can be found in l...</div><p><a href="/proxy/4000/posts/eddl-how-do-we-train-on-limited-edge-devices">Read more</a></p></div><div class="article__info clearfix"><ul class="left-col menu"><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Research">Research</a>
+                href="/proxy/4000/archive.html?tag=Research">Research</a>
             </li><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Edge+computing">Edge computing</a>
+                href="/proxy/4000/archive.html?tag=Edge+computing">Edge computing</a>
             </li></ul><ul class="right-col menu"><li><i class="fas fa-user"></i> <span>Pengzhan Hao</span></li><li><i class="far fa-calendar-alt"></i> <span>Oct 13, 2021</span>
             </li></ul></div><meta itemprop="author" content="Pengzhan Hao"/><meta itemprop="datePublished" content="2021-10-13T16:53:20-04:00">
     <meta itemprop="keywords" content="Research,Edge computing"></div>
-      </article><article class="item" itemscope itemtype="http://schema.org/BlogPosting"><div class="item__image" style="vertical-align: middle"><img class="image" src="/static/2020-09/2020-06-28.png" /></div><div class="item__content">
-          <header><a href="/posts/generate-word-cloud-with-chinese-fenci"><h2 itemprop="headline" class="item__header">Generate Word Cloud Figures with Chinese-Tokenization and WordCloud python libraries</h2></a></header>
+      </article><article class="item" itemscope itemtype="http://schema.org/BlogPosting"><div class="item__image" style="vertical-align: middle"><img class="image" src="/proxy/4000/static/2020-09/2020-06-28.png" /></div><div class="item__content">
+          <header><a href="/proxy/4000/posts/generate-word-cloud-with-chinese-fenci"><h2 itemprop="headline" class="item__header">Generate Word Cloud Figures with Chinese-Tokenization and WordCloud python libraries</h2></a></header>
           <div class="item__description"><div class="article__content" itemprop="description articleBody">Let’s generate a word cloud like this. 
 Don’t understand the language is not a big deal.
-If your written language is based on latin alphabet(or other language has space between words), skip tokenization.</div><p><a href="/posts/generate-word-cloud-with-chinese-fenci">Read more</a></p></div><div class="article__info clearfix"><ul class="left-col menu"><li>
+If your written language is based on latin alphabet(or other language has space between words), skip tokenization.</div><p><a href="/proxy/4000/posts/generate-word-cloud-with-chinese-fenci">Read more</a></p></div><div class="article__info clearfix"><ul class="left-col menu"><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Visualization">Visualization</a>
+                href="/proxy/4000/archive.html?tag=Visualization">Visualization</a>
             </li></ul><ul class="right-col menu"><li><i class="fas fa-user"></i> <span>Pengzhan Hao</span></li><li><i class="far fa-calendar-alt"></i> <span>Sep 15, 2020</span>
             </li></ul></div><meta itemprop="author" content="Pengzhan Hao"/><meta itemprop="datePublished" content="2020-09-15T22:00:14-04:00">
     <meta itemprop="keywords" content="Visualization"></div>
-      </article><article class="item" itemscope itemtype="http://schema.org/BlogPosting"><div class="item__image" style="vertical-align: middle"><img class="image" src="/static/2021-10/Xv6_LS_Command_Output.png" /></div><div class="item__content">
-          <header><a href="/posts/intro-xv6"><h2 itemprop="headline" class="item__header">Xv6 introduction</h2></a></header>
+      </article><article class="item" itemscope itemtype="http://schema.org/BlogPosting"><div class="item__image" style="vertical-align: middle"><img class="image" src="/proxy/4000/static/2021-10/Xv6_LS_Command_Output.png" /></div><div class="item__content">
+          <header><a href="/proxy/4000/posts/intro-xv6"><h2 itemprop="headline" class="item__header">Xv6 introduction</h2></a></header>
           <div class="item__description"><div class="article__content" itemprop="description articleBody">In this post, you will learn a few basic concepts of xv6. Learning path will be closed coupled to first project assignment I gave when I assisted in teaching OS classes.
 Understand system call and know how to implement a simple one will be coved as the first half.
-In the second half of this post, I will discuss a little bit more on how to debug ...</div><p><a href="/posts/intro-xv6">Read more</a></p></div><div class="article__info clearfix"><ul class="left-col menu"><li>
+In the second half of this post, I will discuss a little bit more on how to debug ...</div><p><a href="/proxy/4000/posts/intro-xv6">Read more</a></p></div><div class="article__info clearfix"><ul class="left-col menu"><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Xv6">Xv6</a>
+                href="/proxy/4000/archive.html?tag=Xv6">Xv6</a>
             </li><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Teaching">Teaching</a>
+                href="/proxy/4000/archive.html?tag=Teaching">Teaching</a>
             </li><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Operating+system">Operating system</a>
+                href="/proxy/4000/archive.html?tag=Operating+system">Operating system</a>
             </li></ul><ul class="right-col menu"><li><i class="fas fa-user"></i> <span>Pengzhan Hao</span></li><li><i class="far fa-calendar-alt"></i> <span>Jul 28, 2017</span>
             </li></ul></div><meta itemprop="author" content="Pengzhan Hao"/><meta itemprop="datePublished" content="2017-07-28T14:56:55-04:00">
     <meta itemprop="keywords" content="Xv6,Teaching,Operating system"></div>
       </article><article class="item" itemscope itemtype="http://schema.org/BlogPosting"><div class="item__content">
-          <header><a href="/posts/some-of-my-previews-exper-work"><h2 itemprop="headline" class="item__header">Some of my previews experiment works: 2016</h2></a></header>
-          <div class="item__description"><div class="article__content" itemprop="description articleBody">This blog contains only some basic record of my works. For some details, I will write a unique blog just for some specific topics.</div><p><a href="/posts/some-of-my-previews-exper-work">Read more</a></p></div><div class="article__info clearfix"><ul class="left-col menu"><li>
+          <header><a href="/proxy/4000/posts/some-of-my-previews-exper-work"><h2 itemprop="headline" class="item__header">Some of my previews experiment works: 2016</h2></a></header>
+          <div class="item__description"><div class="article__content" itemprop="description articleBody">This blog contains only some basic record of my works. For some details, I will write a unique blog just for some specific topics.</div><p><a href="/proxy/4000/posts/some-of-my-previews-exper-work">Read more</a></p></div><div class="article__info clearfix"><ul class="left-col menu"><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Research">Research</a>
+                href="/proxy/4000/archive.html?tag=Research">Research</a>
             </li><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Log">Log</a>
+                href="/proxy/4000/archive.html?tag=Log">Log</a>
             </li><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Miscellanies">Miscellanies</a>
+                href="/proxy/4000/archive.html?tag=Miscellanies">Miscellanies</a>
             </li></ul><ul class="right-col menu"><li><i class="fas fa-user"></i> <span>Pengzhan Hao</span></li><li><i class="far fa-calendar-alt"></i> <span>Oct 28, 2016</span>
             </li></ul></div><meta itemprop="author" content="Pengzhan Hao"/><meta itemprop="datePublished" content="2016-10-28T12:27:33-04:00">
     <meta itemprop="keywords" content="Research,Log,Miscellanies"></div>
-      </article><article class="item" itemscope itemtype="http://schema.org/BlogPosting"><div class="item__image" style="vertical-align: middle"><img class="image" src="/static/2021-12/charles-proxy-logo.png" /></div><div class="item__content">
-          <header><a href="/posts/charles-is-not-a-good-tool"><h2 itemprop="headline" class="item__header">Using charles proxy to monitor mobile SSL traffics</h2></a></header>
-          <div class="item__description"><div class="article__content" itemprop="description articleBody">In this blog, I will generally talk about how to use proper tools to monitor SSL traffics of a mobile devices. Currently, I only can dealing with those SSL traffics which use an obviously certification. Some applications may not using system root cert or they doesn’t provide us a method to modify their own certs. For these situation, I still did...</div><p><a href="/posts/charles-is-not-a-good-tool">Read more</a></p></div><div class="article__info clearfix"><ul class="left-col menu"><li>
-              <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Network">Network</a>
-            </li><li>
-              <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Charles+proxy">Charles proxy</a>
-            </li></ul><ul class="right-col menu"><li><i class="fas fa-user"></i> <span>Pengzhan Hao</span></li><li><i class="far fa-calendar-alt"></i> <span>Oct 27, 2016</span>
-            </li></ul></div><meta itemprop="author" content="Pengzhan Hao"/><meta itemprop="datePublished" content="2016-10-27T22:50:33-04:00">
-    <meta itemprop="keywords" content="Network,Charles proxy"></div>
       </article></div>
-</div><div class="layout--home"><div class="pagination"><p>9 post articles, 2 pages.</p>
+</div><div class="layout--home"><div class="pagination"><p>10 post articles, 2 pages.</p>
     <div class="pagination__menu">
       <ul class="menu menu--nowrap"><li><div class="button button--secondary button--circle disabled">
             <i class="fas fa-angle-left"></i>
           </div></li><li>
               <div class="button button--primary button--circle focus"><span>1</span></div>
             </li><li>
-                <a class="button button--secondary button--circle" href="/page2"><span>2</span></a>
-              </li><li><a class="button button--secondary button--circle" href="/page2">
+                <a class="button button--secondary button--circle" href="/proxy/4000/page2"><span>2</span></a>
+              </li><li><a class="button button--secondary button--circle" href="/proxy/4000/page2">
             <i class="fas fa-angle-right"></i>
           </a></li></ul>
     </div>
@@ -585,7 +598,7 @@ In the second half of this post, I will discuss a little bit more on how to debu
             </div></div></div><div class="page__footer d-print-none">
 <footer class="footer py-4 js-page-footer">
   <div class="main"><div itemscope itemtype="http://schema.org/Person">
-      <meta itemprop="name" content="Pengzhan Hao"><meta itemprop="url" content="/"><div class="footer__author-links"><div class="author-links">
+      <meta itemprop="name" content="Pengzhan Hao"><meta itemprop="url" content="/proxy/4000/"><div class="footer__author-links"><div class="author-links">
   <ul class="menu menu--nowrap menu--inline"><li title="Send me an Email.">
       <a class="button button--circle mail-button" itemprop="email" href="mailto:haopengzhan@gmail.com" target="_blank">
         <div class="icon"><svg fill="#000000" width="24px" height="24px" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg">
diff --git a/_site/posts/charles-is-not-a-good-tool.html b/_site/posts/charles-is-not-a-good-tool.html
index d0cb81e..0d8fbd1 100644
--- a/_site/posts/charles-is-not-a-good-tool.html
+++ b/_site/posts/charles-is-not-a-good-tool.html
@@ -4,7 +4,7 @@
 <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no"><title>Using charles proxy to monitor mobile SSL traffics - STSD</title>
 
 <meta name="description" content="In this blog, I will generally talk about how to use proper tools to monitor SSL traffics of a mobile devices. Currently, I only can dealing with those SSL t...">
-<link rel="canonical" href="https://blog.pengzhan.dev/posts/charles-is-not-a-good-tool"><link rel="alternate" type="application/rss+xml" title="STSD" href="/feed.xml"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
+<link rel="canonical" href="https://dev.pengzhan.dev/proxy/4000/posts/charles-is-not-a-good-tool"><link rel="alternate" type="application/rss+xml" title="STSD" href="/proxy/4000/feed.xml"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
 <link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png">
 <link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png">
 <link rel="manifest" href="/assets/site.webmanifest">
@@ -13,7 +13,7 @@
 <meta name="msapplication-TileColor" content="#da532c">
 <meta name="msapplication-config" content="/assets/browserconfig.xml">
 <meta name="theme-color" content="#ffffff">
-<link rel="stylesheet" href="/assets/css/main.css"><link rel="stylesheet" href="/static/2022-09/all.css" >
+<link rel="stylesheet" href="/proxy/4000/assets/css/main.css"><link rel="stylesheet" href="/static/2022-09/all.css" >
 
 <style>
     @import url('https://fonts.googleapis.com/css2?family=Poppins:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;0,800;0,900;1,100;1,200;1,300;1,400;1,500;1,600;1,700;1,800;1,900&family=Roboto+Slab:wght@100..900&family=ZCOOL+XiaoWei&display=swap')
@@ -26,15 +26,15 @@
 <meta property="og:locale" content="en" />
 <meta name="description" content="In this blog, I will generally talk about how to use proper tools to monitor SSL traffics of a mobile devices. Currently, I only can dealing with those SSL traffics which use an obviously certification. Some applications may not using system root cert or they doesn’t provide us a method to modify their own certs. For these situation, I still didn’t find a good solutions for it. But I’ll keep updating this if I get one. My current solution is using AP to forward all SSL traffic to a proxy, charles proxy is my first choice (Prof asked). It’s a non-free software which still update new versions now. So mainly, I’ll talk about how to charles SSL proxy." />
 <meta property="og:description" content="In this blog, I will generally talk about how to use proper tools to monitor SSL traffics of a mobile devices. Currently, I only can dealing with those SSL traffics which use an obviously certification. Some applications may not using system root cert or they doesn’t provide us a method to modify their own certs. For these situation, I still didn’t find a good solutions for it. But I’ll keep updating this if I get one. My current solution is using AP to forward all SSL traffic to a proxy, charles proxy is my first choice (Prof asked). It’s a non-free software which still update new versions now. So mainly, I’ll talk about how to charles SSL proxy." />
-<link rel="canonical" href="https://blog.pengzhan.dev/posts/charles-is-not-a-good-tool" />
-<meta property="og:url" content="https://blog.pengzhan.dev/posts/charles-is-not-a-good-tool" />
+<link rel="canonical" href="https://dev.pengzhan.dev/posts/charles-is-not-a-good-tool" />
+<meta property="og:url" content="https://dev.pengzhan.dev/posts/charles-is-not-a-good-tool" />
 <meta property="og:site_name" content="STSD" />
 <meta property="og:type" content="article" />
 <meta property="article:published_time" content="2016-10-27T22:50:33-04:00" />
 <meta name="twitter:card" content="summary" />
 <meta property="twitter:title" content="Using charles proxy to monitor mobile SSL traffics" />
 <script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"Pengzhan Hao"},"dateModified":"2021-12-11T13:35:39-05:00","datePublished":"2016-10-27T22:50:33-04:00","description":"In this blog, I will generally talk about how to use proper tools to monitor SSL traffics of a mobile devices. Currently, I only can dealing with those SSL traffics which use an obviously certification. Some applications may not using system root cert or they doesn’t provide us a method to modify their own certs. For these situation, I still didn’t find a good solutions for it. But I’ll keep updating this if I get one. My current solution is using AP to forward all SSL traffic to a proxy, charles proxy is my first choice (Prof asked). It’s a non-free software which still update new versions now. So mainly, I’ll talk about how to charles SSL proxy.","headline":"Using charles proxy to monitor mobile SSL traffics","mainEntityOfPage":{"@type":"WebPage","@id":"https://blog.pengzhan.dev/posts/charles-is-not-a-good-tool"},"url":"https://blog.pengzhan.dev/posts/charles-is-not-a-good-tool"}</script>
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"Pengzhan Hao"},"dateModified":"2021-12-11T13:35:39-05:00","datePublished":"2016-10-27T22:50:33-04:00","description":"In this blog, I will generally talk about how to use proper tools to monitor SSL traffics of a mobile devices. Currently, I only can dealing with those SSL traffics which use an obviously certification. Some applications may not using system root cert or they doesn’t provide us a method to modify their own certs. For these situation, I still didn’t find a good solutions for it. But I’ll keep updating this if I get one. My current solution is using AP to forward all SSL traffic to a proxy, charles proxy is my first choice (Prof asked). It’s a non-free software which still update new versions now. So mainly, I’ll talk about how to charles SSL proxy.","headline":"Using charles proxy to monitor mobile SSL traffics","mainEntityOfPage":{"@type":"WebPage","@id":"https://dev.pengzhan.dev/posts/charles-is-not-a-good-tool"},"url":"https://dev.pengzhan.dev/posts/charles-is-not-a-good-tool"}</script>
 <!-- End Jekyll SEO tag -->
 
 <!-- end custom head snippets -->
@@ -271,7 +271,7 @@
         }
       },
       paths: {
-        search_js: '/assets/search.js'
+        search_js: '/proxy/4000/assets/search.js'
       }
     };
     window.TEXT_VARIABLES = TEXT_VARIABLES;
@@ -426,9 +426,9 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
 </g>
 </svg>
 -->
-          <img src="/assets/favicon-32x32.png" style="margin-right: 5px;"><a title="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
-" href="/">STSD</a></div><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></div><nav class="navigation">
-        <ul><li class="navigation__item"><a href="/archive.html">Archive</a></li><li class="navigation__item"><a href="https://pengzhan.dev">About</a></li><li><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></li></ul>
+          <img src="/proxy/4000/assets/favicon-32x32.png" style="margin-right: 5px;"><a title="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
+" href="/proxy/4000/">STSD</a></div><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></div><nav class="navigation">
+        <ul><li class="navigation__item"><a href="/proxy/4000/archive.html">Archive</a></li><li class="navigation__item"><a href="https://pengzhan.dev">About</a></li><li><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></li></ul>
       </nav></div>
   </header>
 </div><div class="page__content"><div class ="main"><div class="grid grid--reverse">
@@ -441,10 +441,10 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
 <!-- end custom main top snippet -->
 <article itemscope itemtype="http://schema.org/Article"><div class="article__header"><header><h1>Using charles proxy to monitor mobile SSL traffics</h1></header></div><meta itemprop="headline" content="Using charles proxy to monitor mobile SSL traffics"><div class="article__info clearfix"><ul class="left-col menu"><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Network">Network</a>
+                href="/proxy/4000/archive.html?tag=Network">Network</a>
             </li><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Charles+proxy">Charles proxy</a>
+                href="/proxy/4000/archive.html?tag=Charles+proxy">Charles proxy</a>
             </li></ul><ul class="right-col menu"><li><i class="fas fa-user"></i> <span>Pengzhan Hao</span></li><li><i class="far fa-calendar-alt"></i> <span>Oct 27, 2016</span>
             </li></ul></div><meta itemprop="author" content="Pengzhan Hao"/><meta itemprop="datePublished" content="2016-10-27T22:50:33-04:00">
     <meta itemprop="keywords" content="Network,Charles proxy"><div class="js-article-content"><div class="layout--article"><!-- start custom article top snippet -->
@@ -483,9 +483,9 @@ You also need to save charles Root Certificate, it also contains in the same men
 </div><section class="article__sharing d-print-none"></section><div class="d-print-none"><footer class="article__footer"><meta itemprop="dateModified" content="2016-10-27T22:50:33-04:00"><!-- start custom article footer snippet -->
 
 <!-- end custom article footer snippet -->
-<div class="article__subscribe"><div class="subscribe"><i class="fas fa-rss"></i> <a type="application/rss+xml" href="/feed.xml">Subscribe</a></div>
+<div class="article__subscribe"><div class="subscribe"><i class="fas fa-rss"></i> <a type="application/rss+xml" href="/proxy/4000/feed.xml">Subscribe</a></div>
 </div><div class="article__license"></div></footer>
-<div class="article__section-navigator clearfix"><div class="previous"><span>PREVIOUS</span><a href="/posts/welcome-to-my-blog">STSD: Stop Talking Start Doing</a></div><div class="next"><span>NEXT</span><a href="/posts/some-of-my-previews-exper-work">Some of my previews experiment works: 2016</a></div></div></div>
+<div class="article__section-navigator clearfix"><div class="previous"><span>PREVIOUS</span><a href="/proxy/4000/posts/welcome-to-my-blog">STSD: Stop Talking Start Doing</a></div><div class="next"><span>NEXT</span><a href="/proxy/4000/posts/some-of-my-previews-exper-work">Some of my previews experiment works: 2016</a></div></div></div>
 
 </div>
 
@@ -521,7 +521,7 @@ You also need to save charles Root Certificate, it also contains in the same men
             </div></div></div><div class="page__footer d-print-none">
 <footer class="footer py-4 js-page-footer">
   <div class="main"><div itemscope itemtype="http://schema.org/Person">
-      <meta itemprop="name" content="Pengzhan Hao"><meta itemprop="url" content="/"><div class="footer__author-links"><div class="author-links">
+      <meta itemprop="name" content="Pengzhan Hao"><meta itemprop="url" content="/proxy/4000/"><div class="footer__author-links"><div class="author-links">
   <ul class="menu menu--nowrap menu--inline"><li title="Send me an Email.">
       <a class="button button--circle mail-button" itemprop="email" href="mailto:haopengzhan@gmail.com" target="_blank">
         <div class="icon"><svg fill="#000000" width="24px" height="24px" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg">
diff --git a/_site/posts/eddl-how-do-we-train-on-limited-edge-devices-part2.html b/_site/posts/eddl-how-do-we-train-on-limited-edge-devices-part2.html
index 06c043d..b087668 100644
--- a/_site/posts/eddl-how-do-we-train-on-limited-edge-devices-part2.html
+++ b/_site/posts/eddl-how-do-we-train-on-limited-edge-devices-part2.html
@@ -4,7 +4,7 @@
 <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no"><title>EDDL: How do we train neural networks on limited edge devices - PART 2 - STSD</title>
 
 <meta name="description" content="In the last post, part1, our idea of distributed learning on edge environment was generally addressed.I introduced the reason why edge distributed learning i...">
-<link rel="canonical" href="https://blog.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices-part2"><link rel="alternate" type="application/rss+xml" title="STSD" href="/feed.xml"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
+<link rel="canonical" href="https://dev.pengzhan.dev/proxy/4000/posts/eddl-how-do-we-train-on-limited-edge-devices-part2"><link rel="alternate" type="application/rss+xml" title="STSD" href="/proxy/4000/feed.xml"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
 <link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png">
 <link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png">
 <link rel="manifest" href="/assets/site.webmanifest">
@@ -13,7 +13,7 @@
 <meta name="msapplication-TileColor" content="#da532c">
 <meta name="msapplication-config" content="/assets/browserconfig.xml">
 <meta name="theme-color" content="#ffffff">
-<link rel="stylesheet" href="/assets/css/main.css"><link rel="stylesheet" href="/static/2022-09/all.css" >
+<link rel="stylesheet" href="/proxy/4000/assets/css/main.css"><link rel="stylesheet" href="/static/2022-09/all.css" >
 
 <style>
     @import url('https://fonts.googleapis.com/css2?family=Poppins:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;0,800;0,900;1,100;1,200;1,300;1,400;1,500;1,600;1,700;1,800;1,900&family=Roboto+Slab:wght@100..900&family=ZCOOL+XiaoWei&display=swap')
@@ -26,15 +26,15 @@
 <meta property="og:locale" content="en" />
 <meta name="description" content="In the last post, part1, our idea of distributed learning on edge environment was generally addressed. I introduced the reason why edge distributed learning is needed and what improvements it can achieve. In this post, I will talk about our motivation study and how our framework works. How does data support us training on edge? Before designing and implementing our framework, we first need confirmation that training on edge resource-limited devices is worthwhile. We were using a malware detection neural network to show why a small, customized neural network is better. We collected 32000+ mobile apps feature as global data. With these data records, we trained a multilayer perceptron called “PerNet” to determine whether a given feature belongs to a benign or malware app. We called this detection. As well, PerNet can also classify malware apps into different types of attacks. We called this classification. The global model can achieve 93% above recall rate and 96.93% above accuracy. With all these data, we selected two community app usage sub-dataset for local model generations. Large categories (Scenario 1) We chose the 5 largest categories of apps, including entertainment, tools, brain&amp;Puzzle, Lifestyle, and Education, as well as the 5 largest malware categories. All together, 12000+ apps were included in this sub-dataset, almost 50 to 50 between benign and malware. Campus-community categories (Scenario 2) We chose the 5 most downloaded categories from college students as benign groups, as well as a similar amount of 5 malware categories. To ensure that malware apps are included in 5 benign categories, we also considered synthesizing some other malware apps within categories of 5 most downloaded(benign) categories. With these two types of sub-dataset, we used the same PerNet to generate multiple local models. Under each scenarios experiment, we compared global and local models on the preserved test dataset. In all classification performances, local beat global in every scenario. In detection performances, local also share the same accuracy as global does. In summary, local models were trained on special occasions. Under the same circumstance, a global model can achieve no better accuracy than local models. The reason why local is better might be because of overfitting. I believe this issue also be considered in the machine learning communities that they brought transfer learning, a technique to optimize global models to special scenarios but performing more training to a global model once it’s shipped to local. Design and Implementation Overall design The basic EDDL distributed training setup consists of 3 parts. EDDL training cluster, a device cluster that consists of edge or mobile devices that are participating in training. EDDL manager, the initial driver program that works as collect training data, relay data to training devices and initial training clusters. Training data entry (TDE), a data storage for all training data. Dynamic training data distribution Existing distributed DNN training solutions usually statically partition training data among workers. It can be a problem when the training node joins and exits. We designed our framework that can dynamically distribute training data during learning. Before every training batch started, a batch of TDE will be sent to devices. In our experiments, we found that by applying this design, overall training time was shortened by doing. Especially in large amount devices cases, this optimization can be 50% less than statically divided. Scaling up cluster size Our framework was designed to have both sync and async parameter aggregation. Asynchronous aggregation can allow a high outcome of training batch but with a sacrifice or converge time. Synchronous aggregation allows a quick converge time in epochs, however can’t ensure performance when there’s a struggler worker. As showed in experiments, we chose sync as default because the converging time is dominant in overall training time. But, we also considered the possibilities of that async with more workers can achieve similar overall training time. We introduced a formula to determine whether adding more training nodes can help or not. Here we used bandwidth usage coefficient (BUC) as [BUC = \dfrac{n}{T_{sync}}] In this formula, \(n\) is the number of devices, and \(T_{sync}\) is the transmission time of parameters. With an increasing number of workers, n increase linearly but transmission time does not. When \(BUC\) increases, the cluster can speed up training time by adding workers. Otherwise, adding more workers won’t help with overall training time. Adaptive leader role splitting The idea of role splitting is simple that a device can work as a worker as well leader. The advantage of doing this is straightforward that we can transfer 1 less parameter and training time will be shortened. However, in our current settings, it can’t perform much better help since only 1 leader role is in a cluster. We can benefit from this in our future works. Overall architecture Details were given in the image. Prototype hardware and software EDDL was designed to be run on two single-board computer embedded platforms. One such platform is ODROID-XU4, which is equipped with a 2.1/1.4 GHz 32-bit ARM processor and 2GB memory. The other platform is the Raspberry Pi 3 Model B board, which comes with an ARM 1.2 GHz 64-bit quad-core processor and 1GB memory. The operating system running on the above platforms is Ubuntu 18.04 with Linux kernel 4.14. We used Dlib, a C++ library that provides implementations for a wide range of machine learning algorithms. We chose the Dlib library because it is written in C/C++, and can be easily and natively used in embedded devices." />
 <meta property="og:description" content="In the last post, part1, our idea of distributed learning on edge environment was generally addressed. I introduced the reason why edge distributed learning is needed and what improvements it can achieve. In this post, I will talk about our motivation study and how our framework works. How does data support us training on edge? Before designing and implementing our framework, we first need confirmation that training on edge resource-limited devices is worthwhile. We were using a malware detection neural network to show why a small, customized neural network is better. We collected 32000+ mobile apps feature as global data. With these data records, we trained a multilayer perceptron called “PerNet” to determine whether a given feature belongs to a benign or malware app. We called this detection. As well, PerNet can also classify malware apps into different types of attacks. We called this classification. The global model can achieve 93% above recall rate and 96.93% above accuracy. With all these data, we selected two community app usage sub-dataset for local model generations. Large categories (Scenario 1) We chose the 5 largest categories of apps, including entertainment, tools, brain&amp;Puzzle, Lifestyle, and Education, as well as the 5 largest malware categories. All together, 12000+ apps were included in this sub-dataset, almost 50 to 50 between benign and malware. Campus-community categories (Scenario 2) We chose the 5 most downloaded categories from college students as benign groups, as well as a similar amount of 5 malware categories. To ensure that malware apps are included in 5 benign categories, we also considered synthesizing some other malware apps within categories of 5 most downloaded(benign) categories. With these two types of sub-dataset, we used the same PerNet to generate multiple local models. Under each scenarios experiment, we compared global and local models on the preserved test dataset. In all classification performances, local beat global in every scenario. In detection performances, local also share the same accuracy as global does. In summary, local models were trained on special occasions. Under the same circumstance, a global model can achieve no better accuracy than local models. The reason why local is better might be because of overfitting. I believe this issue also be considered in the machine learning communities that they brought transfer learning, a technique to optimize global models to special scenarios but performing more training to a global model once it’s shipped to local. Design and Implementation Overall design The basic EDDL distributed training setup consists of 3 parts. EDDL training cluster, a device cluster that consists of edge or mobile devices that are participating in training. EDDL manager, the initial driver program that works as collect training data, relay data to training devices and initial training clusters. Training data entry (TDE), a data storage for all training data. Dynamic training data distribution Existing distributed DNN training solutions usually statically partition training data among workers. It can be a problem when the training node joins and exits. We designed our framework that can dynamically distribute training data during learning. Before every training batch started, a batch of TDE will be sent to devices. In our experiments, we found that by applying this design, overall training time was shortened by doing. Especially in large amount devices cases, this optimization can be 50% less than statically divided. Scaling up cluster size Our framework was designed to have both sync and async parameter aggregation. Asynchronous aggregation can allow a high outcome of training batch but with a sacrifice or converge time. Synchronous aggregation allows a quick converge time in epochs, however can’t ensure performance when there’s a struggler worker. As showed in experiments, we chose sync as default because the converging time is dominant in overall training time. But, we also considered the possibilities of that async with more workers can achieve similar overall training time. We introduced a formula to determine whether adding more training nodes can help or not. Here we used bandwidth usage coefficient (BUC) as [BUC = \dfrac{n}{T_{sync}}] In this formula, \(n\) is the number of devices, and \(T_{sync}\) is the transmission time of parameters. With an increasing number of workers, n increase linearly but transmission time does not. When \(BUC\) increases, the cluster can speed up training time by adding workers. Otherwise, adding more workers won’t help with overall training time. Adaptive leader role splitting The idea of role splitting is simple that a device can work as a worker as well leader. The advantage of doing this is straightforward that we can transfer 1 less parameter and training time will be shortened. However, in our current settings, it can’t perform much better help since only 1 leader role is in a cluster. We can benefit from this in our future works. Overall architecture Details were given in the image. Prototype hardware and software EDDL was designed to be run on two single-board computer embedded platforms. One such platform is ODROID-XU4, which is equipped with a 2.1/1.4 GHz 32-bit ARM processor and 2GB memory. The other platform is the Raspberry Pi 3 Model B board, which comes with an ARM 1.2 GHz 64-bit quad-core processor and 1GB memory. The operating system running on the above platforms is Ubuntu 18.04 with Linux kernel 4.14. We used Dlib, a C++ library that provides implementations for a wide range of machine learning algorithms. We chose the Dlib library because it is written in C/C++, and can be easily and natively used in embedded devices." />
-<link rel="canonical" href="https://blog.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices-part2" />
-<meta property="og:url" content="https://blog.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices-part2" />
+<link rel="canonical" href="https://dev.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices-part2" />
+<meta property="og:url" content="https://dev.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices-part2" />
 <meta property="og:site_name" content="STSD" />
 <meta property="og:type" content="article" />
 <meta property="article:published_time" content="2021-10-31T13:01:14-04:00" />
 <meta name="twitter:card" content="summary" />
 <meta property="twitter:title" content="EDDL: How do we train neural networks on limited edge devices - PART 2" />
 <script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"Pengzhan Hao"},"dateModified":"2021-12-11T13:35:39-05:00","datePublished":"2021-10-31T13:01:14-04:00","description":"In the last post, part1, our idea of distributed learning on edge environment was generally addressed. I introduced the reason why edge distributed learning is needed and what improvements it can achieve. In this post, I will talk about our motivation study and how our framework works. How does data support us training on edge? Before designing and implementing our framework, we first need confirmation that training on edge resource-limited devices is worthwhile. We were using a malware detection neural network to show why a small, customized neural network is better. We collected 32000+ mobile apps feature as global data. With these data records, we trained a multilayer perceptron called “PerNet” to determine whether a given feature belongs to a benign or malware app. We called this detection. As well, PerNet can also classify malware apps into different types of attacks. We called this classification. The global model can achieve 93% above recall rate and 96.93% above accuracy. With all these data, we selected two community app usage sub-dataset for local model generations. Large categories (Scenario 1) We chose the 5 largest categories of apps, including entertainment, tools, brain&amp;Puzzle, Lifestyle, and Education, as well as the 5 largest malware categories. All together, 12000+ apps were included in this sub-dataset, almost 50 to 50 between benign and malware. Campus-community categories (Scenario 2) We chose the 5 most downloaded categories from college students as benign groups, as well as a similar amount of 5 malware categories. To ensure that malware apps are included in 5 benign categories, we also considered synthesizing some other malware apps within categories of 5 most downloaded(benign) categories. With these two types of sub-dataset, we used the same PerNet to generate multiple local models. Under each scenarios experiment, we compared global and local models on the preserved test dataset. In all classification performances, local beat global in every scenario. In detection performances, local also share the same accuracy as global does. In summary, local models were trained on special occasions. Under the same circumstance, a global model can achieve no better accuracy than local models. The reason why local is better might be because of overfitting. I believe this issue also be considered in the machine learning communities that they brought transfer learning, a technique to optimize global models to special scenarios but performing more training to a global model once it’s shipped to local. Design and Implementation Overall design The basic EDDL distributed training setup consists of 3 parts. EDDL training cluster, a device cluster that consists of edge or mobile devices that are participating in training. EDDL manager, the initial driver program that works as collect training data, relay data to training devices and initial training clusters. Training data entry (TDE), a data storage for all training data. Dynamic training data distribution Existing distributed DNN training solutions usually statically partition training data among workers. It can be a problem when the training node joins and exits. We designed our framework that can dynamically distribute training data during learning. Before every training batch started, a batch of TDE will be sent to devices. In our experiments, we found that by applying this design, overall training time was shortened by doing. Especially in large amount devices cases, this optimization can be 50% less than statically divided. Scaling up cluster size Our framework was designed to have both sync and async parameter aggregation. Asynchronous aggregation can allow a high outcome of training batch but with a sacrifice or converge time. Synchronous aggregation allows a quick converge time in epochs, however can’t ensure performance when there’s a struggler worker. As showed in experiments, we chose sync as default because the converging time is dominant in overall training time. But, we also considered the possibilities of that async with more workers can achieve similar overall training time. We introduced a formula to determine whether adding more training nodes can help or not. Here we used bandwidth usage coefficient (BUC) as [BUC = \\dfrac{n}{T_{sync}}] In this formula, \\(n\\) is the number of devices, and \\(T_{sync}\\) is the transmission time of parameters. With an increasing number of workers, n increase linearly but transmission time does not. When \\(BUC\\) increases, the cluster can speed up training time by adding workers. Otherwise, adding more workers won’t help with overall training time. Adaptive leader role splitting The idea of role splitting is simple that a device can work as a worker as well leader. The advantage of doing this is straightforward that we can transfer 1 less parameter and training time will be shortened. However, in our current settings, it can’t perform much better help since only 1 leader role is in a cluster. We can benefit from this in our future works. Overall architecture Details were given in the image. Prototype hardware and software EDDL was designed to be run on two single-board computer embedded platforms. One such platform is ODROID-XU4, which is equipped with a 2.1/1.4 GHz 32-bit ARM processor and 2GB memory. The other platform is the Raspberry Pi 3 Model B board, which comes with an ARM 1.2 GHz 64-bit quad-core processor and 1GB memory. The operating system running on the above platforms is Ubuntu 18.04 with Linux kernel 4.14. We used Dlib, a C++ library that provides implementations for a wide range of machine learning algorithms. We chose the Dlib library because it is written in C/C++, and can be easily and natively used in embedded devices.","headline":"EDDL: How do we train neural networks on limited edge devices - PART 2","mainEntityOfPage":{"@type":"WebPage","@id":"https://blog.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices-part2"},"url":"https://blog.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices-part2"}</script>
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"Pengzhan Hao"},"dateModified":"2021-12-11T13:35:39-05:00","datePublished":"2021-10-31T13:01:14-04:00","description":"In the last post, part1, our idea of distributed learning on edge environment was generally addressed. I introduced the reason why edge distributed learning is needed and what improvements it can achieve. In this post, I will talk about our motivation study and how our framework works. How does data support us training on edge? Before designing and implementing our framework, we first need confirmation that training on edge resource-limited devices is worthwhile. We were using a malware detection neural network to show why a small, customized neural network is better. We collected 32000+ mobile apps feature as global data. With these data records, we trained a multilayer perceptron called “PerNet” to determine whether a given feature belongs to a benign or malware app. We called this detection. As well, PerNet can also classify malware apps into different types of attacks. We called this classification. The global model can achieve 93% above recall rate and 96.93% above accuracy. With all these data, we selected two community app usage sub-dataset for local model generations. Large categories (Scenario 1) We chose the 5 largest categories of apps, including entertainment, tools, brain&amp;Puzzle, Lifestyle, and Education, as well as the 5 largest malware categories. All together, 12000+ apps were included in this sub-dataset, almost 50 to 50 between benign and malware. Campus-community categories (Scenario 2) We chose the 5 most downloaded categories from college students as benign groups, as well as a similar amount of 5 malware categories. To ensure that malware apps are included in 5 benign categories, we also considered synthesizing some other malware apps within categories of 5 most downloaded(benign) categories. With these two types of sub-dataset, we used the same PerNet to generate multiple local models. Under each scenarios experiment, we compared global and local models on the preserved test dataset. In all classification performances, local beat global in every scenario. In detection performances, local also share the same accuracy as global does. In summary, local models were trained on special occasions. Under the same circumstance, a global model can achieve no better accuracy than local models. The reason why local is better might be because of overfitting. I believe this issue also be considered in the machine learning communities that they brought transfer learning, a technique to optimize global models to special scenarios but performing more training to a global model once it’s shipped to local. Design and Implementation Overall design The basic EDDL distributed training setup consists of 3 parts. EDDL training cluster, a device cluster that consists of edge or mobile devices that are participating in training. EDDL manager, the initial driver program that works as collect training data, relay data to training devices and initial training clusters. Training data entry (TDE), a data storage for all training data. Dynamic training data distribution Existing distributed DNN training solutions usually statically partition training data among workers. It can be a problem when the training node joins and exits. We designed our framework that can dynamically distribute training data during learning. Before every training batch started, a batch of TDE will be sent to devices. In our experiments, we found that by applying this design, overall training time was shortened by doing. Especially in large amount devices cases, this optimization can be 50% less than statically divided. Scaling up cluster size Our framework was designed to have both sync and async parameter aggregation. Asynchronous aggregation can allow a high outcome of training batch but with a sacrifice or converge time. Synchronous aggregation allows a quick converge time in epochs, however can’t ensure performance when there’s a struggler worker. As showed in experiments, we chose sync as default because the converging time is dominant in overall training time. But, we also considered the possibilities of that async with more workers can achieve similar overall training time. We introduced a formula to determine whether adding more training nodes can help or not. Here we used bandwidth usage coefficient (BUC) as [BUC = \\dfrac{n}{T_{sync}}] In this formula, \\(n\\) is the number of devices, and \\(T_{sync}\\) is the transmission time of parameters. With an increasing number of workers, n increase linearly but transmission time does not. When \\(BUC\\) increases, the cluster can speed up training time by adding workers. Otherwise, adding more workers won’t help with overall training time. Adaptive leader role splitting The idea of role splitting is simple that a device can work as a worker as well leader. The advantage of doing this is straightforward that we can transfer 1 less parameter and training time will be shortened. However, in our current settings, it can’t perform much better help since only 1 leader role is in a cluster. We can benefit from this in our future works. Overall architecture Details were given in the image. Prototype hardware and software EDDL was designed to be run on two single-board computer embedded platforms. One such platform is ODROID-XU4, which is equipped with a 2.1/1.4 GHz 32-bit ARM processor and 2GB memory. The other platform is the Raspberry Pi 3 Model B board, which comes with an ARM 1.2 GHz 64-bit quad-core processor and 1GB memory. The operating system running on the above platforms is Ubuntu 18.04 with Linux kernel 4.14. We used Dlib, a C++ library that provides implementations for a wide range of machine learning algorithms. We chose the Dlib library because it is written in C/C++, and can be easily and natively used in embedded devices.","headline":"EDDL: How do we train neural networks on limited edge devices - PART 2","mainEntityOfPage":{"@type":"WebPage","@id":"https://dev.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices-part2"},"url":"https://dev.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices-part2"}</script>
 <!-- End Jekyll SEO tag -->
 
 <!-- end custom head snippets -->
@@ -271,7 +271,7 @@
         }
       },
       paths: {
-        search_js: '/assets/search.js'
+        search_js: '/proxy/4000/assets/search.js'
       }
     };
     window.TEXT_VARIABLES = TEXT_VARIABLES;
@@ -426,9 +426,9 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
 </g>
 </svg>
 -->
-          <img src="/assets/favicon-32x32.png" style="margin-right: 5px;"><a title="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
-" href="/">STSD</a></div><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></div><nav class="navigation">
-        <ul><li class="navigation__item"><a href="/archive.html">Archive</a></li><li class="navigation__item"><a href="https://pengzhan.dev">About</a></li><li><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></li></ul>
+          <img src="/proxy/4000/assets/favicon-32x32.png" style="margin-right: 5px;"><a title="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
+" href="/proxy/4000/">STSD</a></div><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></div><nav class="navigation">
+        <ul><li class="navigation__item"><a href="/proxy/4000/archive.html">Archive</a></li><li class="navigation__item"><a href="https://pengzhan.dev">About</a></li><li><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></li></ul>
       </nav></div>
   </header>
 </div><div class="page__content"><div class ="main"><div class="grid grid--reverse">
@@ -441,10 +441,10 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
 <!-- end custom main top snippet -->
 <article itemscope itemtype="http://schema.org/Article"><div class="article__header"><header><h1>EDDL: How do we train neural networks on limited edge devices - PART 2</h1></header></div><meta itemprop="headline" content="EDDL: How do we train neural networks on limited edge devices - PART 2"><div class="article__info clearfix"><ul class="left-col menu"><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Research">Research</a>
+                href="/proxy/4000/archive.html?tag=Research">Research</a>
             </li><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Edge+computing">Edge computing</a>
+                href="/proxy/4000/archive.html?tag=Edge+computing">Edge computing</a>
             </li></ul><ul class="right-col menu"><li><i class="fas fa-user"></i> <span>Pengzhan Hao</span></li><li><i class="far fa-calendar-alt"></i> <span>Oct 31, 2021</span>
             </li></ul></div><meta itemprop="author" content="Pengzhan Hao"/><meta itemprop="datePublished" content="2021-10-31T13:01:14-04:00">
     <meta itemprop="keywords" content="Research,Edge computing"><div class="js-article-content"><div class="layout--article"><!-- start custom article top snippet -->
@@ -558,9 +558,9 @@ We chose the Dlib library because it is written in C/C++, and can be easily and
 </div><section class="article__sharing d-print-none"></section><div class="d-print-none"><footer class="article__footer"><meta itemprop="dateModified" content="2021-10-31T13:01:14-04:00"><!-- start custom article footer snippet -->
 
 <!-- end custom article footer snippet -->
-<div class="article__subscribe"><div class="subscribe"><i class="fas fa-rss"></i> <a type="application/rss+xml" href="/feed.xml">Subscribe</a></div>
+<div class="article__subscribe"><div class="subscribe"><i class="fas fa-rss"></i> <a type="application/rss+xml" href="/proxy/4000/feed.xml">Subscribe</a></div>
 </div><div class="article__license"></div></footer>
-<div class="article__section-navigator clearfix"><div class="previous"><span>PREVIOUS</span><a href="/posts/eddl-how-do-we-train-on-limited-edge-devices">EDDL: How do we train neural networks on limited edge devices - PART 1</a></div><div class="next"><span>NEXT</span><a href="/posts/cs350-labs">Labs of CS350</a></div></div></div>
+<div class="article__section-navigator clearfix"><div class="previous"><span>PREVIOUS</span><a href="/proxy/4000/posts/eddl-how-do-we-train-on-limited-edge-devices">EDDL: How do we train neural networks on limited edge devices - PART 1</a></div><div class="next"><span>NEXT</span><a href="/proxy/4000/posts/cs350-labs">Labs of CS350</a></div></div></div>
 
 </div>
 
@@ -596,7 +596,7 @@ We chose the Dlib library because it is written in C/C++, and can be easily and
             </div></div></div><div class="page__footer d-print-none">
 <footer class="footer py-4 js-page-footer">
   <div class="main"><div itemscope itemtype="http://schema.org/Person">
-      <meta itemprop="name" content="Pengzhan Hao"><meta itemprop="url" content="/"><div class="footer__author-links"><div class="author-links">
+      <meta itemprop="name" content="Pengzhan Hao"><meta itemprop="url" content="/proxy/4000/"><div class="footer__author-links"><div class="author-links">
   <ul class="menu menu--nowrap menu--inline"><li title="Send me an Email.">
       <a class="button button--circle mail-button" itemprop="email" href="mailto:haopengzhan@gmail.com" target="_blank">
         <div class="icon"><svg fill="#000000" width="24px" height="24px" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg">
diff --git a/_site/posts/eddl-how-do-we-train-on-limited-edge-devices.html b/_site/posts/eddl-how-do-we-train-on-limited-edge-devices.html
index dcd2c3e..3d197d5 100644
--- a/_site/posts/eddl-how-do-we-train-on-limited-edge-devices.html
+++ b/_site/posts/eddl-how-do-we-train-on-limited-edge-devices.html
@@ -4,7 +4,7 @@
 <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no"><title>EDDL: How do we train neural networks on limited edge devices - PART 1 - STSD</title>
 
 <meta name="description" content="This post introduces our previous milestone in project “Edge trainer”, as the paper “EDDL: A Distributed Deep Learning System for Resource-limited Edge Compu...">
-<link rel="canonical" href="https://blog.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices"><link rel="alternate" type="application/rss+xml" title="STSD" href="/feed.xml"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
+<link rel="canonical" href="https://dev.pengzhan.dev/proxy/4000/posts/eddl-how-do-we-train-on-limited-edge-devices"><link rel="alternate" type="application/rss+xml" title="STSD" href="/proxy/4000/feed.xml"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
 <link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png">
 <link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png">
 <link rel="manifest" href="/assets/site.webmanifest">
@@ -13,7 +13,7 @@
 <meta name="msapplication-TileColor" content="#da532c">
 <meta name="msapplication-config" content="/assets/browserconfig.xml">
 <meta name="theme-color" content="#ffffff">
-<link rel="stylesheet" href="/assets/css/main.css"><link rel="stylesheet" href="/static/2022-09/all.css" >
+<link rel="stylesheet" href="/proxy/4000/assets/css/main.css"><link rel="stylesheet" href="/static/2022-09/all.css" >
 
 <style>
     @import url('https://fonts.googleapis.com/css2?family=Poppins:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;0,800;0,900;1,100;1,200;1,300;1,400;1,500;1,600;1,700;1,800;1,900&family=Roboto+Slab:wght@100..900&family=ZCOOL+XiaoWei&display=swap')
@@ -26,15 +26,15 @@
 <meta property="og:locale" content="en" />
 <meta name="description" content="This post introduces our previous milestone in project “Edge trainer”, as the paper “EDDL: A Distributed Deep Learning System for Resource-limited Edge Computing Environment.” was published. As the first part of the introductions, I focus only on the motivation and summary of our works. More details in design and implementation can be found in late posts." />
 <meta property="og:description" content="This post introduces our previous milestone in project “Edge trainer”, as the paper “EDDL: A Distributed Deep Learning System for Resource-limited Edge Computing Environment.” was published. As the first part of the introductions, I focus only on the motivation and summary of our works. More details in design and implementation can be found in late posts." />
-<link rel="canonical" href="https://blog.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices" />
-<meta property="og:url" content="https://blog.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices" />
+<link rel="canonical" href="https://dev.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices" />
+<meta property="og:url" content="https://dev.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices" />
 <meta property="og:site_name" content="STSD" />
 <meta property="og:type" content="article" />
 <meta property="article:published_time" content="2021-10-13T16:53:20-04:00" />
 <meta name="twitter:card" content="summary" />
 <meta property="twitter:title" content="EDDL: How do we train neural networks on limited edge devices - PART 1" />
 <script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"Pengzhan Hao"},"dateModified":"2021-12-11T13:35:39-05:00","datePublished":"2021-10-13T16:53:20-04:00","description":"This post introduces our previous milestone in project “Edge trainer”, as the paper “EDDL: A Distributed Deep Learning System for Resource-limited Edge Computing Environment.” was published. As the first part of the introductions, I focus only on the motivation and summary of our works. More details in design and implementation can be found in late posts.","headline":"EDDL: How do we train neural networks on limited edge devices - PART 1","mainEntityOfPage":{"@type":"WebPage","@id":"https://blog.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices"},"url":"https://blog.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices"}</script>
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"Pengzhan Hao"},"dateModified":"2021-12-11T13:35:39-05:00","datePublished":"2021-10-13T16:53:20-04:00","description":"This post introduces our previous milestone in project “Edge trainer”, as the paper “EDDL: A Distributed Deep Learning System for Resource-limited Edge Computing Environment.” was published. As the first part of the introductions, I focus only on the motivation and summary of our works. More details in design and implementation can be found in late posts.","headline":"EDDL: How do we train neural networks on limited edge devices - PART 1","mainEntityOfPage":{"@type":"WebPage","@id":"https://dev.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices"},"url":"https://dev.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices"}</script>
 <!-- End Jekyll SEO tag -->
 
 <!-- end custom head snippets -->
@@ -271,7 +271,7 @@
         }
       },
       paths: {
-        search_js: '/assets/search.js'
+        search_js: '/proxy/4000/assets/search.js'
       }
     };
     window.TEXT_VARIABLES = TEXT_VARIABLES;
@@ -426,9 +426,9 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
 </g>
 </svg>
 -->
-          <img src="/assets/favicon-32x32.png" style="margin-right: 5px;"><a title="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
-" href="/">STSD</a></div><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></div><nav class="navigation">
-        <ul><li class="navigation__item"><a href="/archive.html">Archive</a></li><li class="navigation__item"><a href="https://pengzhan.dev">About</a></li><li><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></li></ul>
+          <img src="/proxy/4000/assets/favicon-32x32.png" style="margin-right: 5px;"><a title="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
+" href="/proxy/4000/">STSD</a></div><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></div><nav class="navigation">
+        <ul><li class="navigation__item"><a href="/proxy/4000/archive.html">Archive</a></li><li class="navigation__item"><a href="https://pengzhan.dev">About</a></li><li><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></li></ul>
       </nav></div>
   </header>
 </div><div class="page__content"><div class ="main"><div class="grid grid--reverse">
@@ -441,10 +441,10 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
 <!-- end custom main top snippet -->
 <article itemscope itemtype="http://schema.org/Article"><div class="article__header"><header><h1>EDDL: How do we train neural networks on limited edge devices - PART 1</h1></header></div><meta itemprop="headline" content="EDDL: How do we train neural networks on limited edge devices - PART 1"><div class="article__info clearfix"><ul class="left-col menu"><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Research">Research</a>
+                href="/proxy/4000/archive.html?tag=Research">Research</a>
             </li><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Edge+computing">Edge computing</a>
+                href="/proxy/4000/archive.html?tag=Edge+computing">Edge computing</a>
             </li></ul><ul class="right-col menu"><li><i class="fas fa-user"></i> <span>Pengzhan Hao</span></li><li><i class="far fa-calendar-alt"></i> <span>Oct 13, 2021</span>
             </li></ul></div><meta itemprop="author" content="Pengzhan Hao"/><meta itemprop="datePublished" content="2021-10-13T16:53:20-04:00">
     <meta itemprop="keywords" content="Research,Edge computing"><div class="js-article-content"><div class="layout--article"><!-- start custom article top snippet -->
@@ -531,9 +531,9 @@ Smartly schedule work balance and handle join/exit issues also need under consid
 </div><section class="article__sharing d-print-none"></section><div class="d-print-none"><footer class="article__footer"><meta itemprop="dateModified" content="2021-10-13T16:53:20-04:00"><!-- start custom article footer snippet -->
 
 <!-- end custom article footer snippet -->
-<div class="article__subscribe"><div class="subscribe"><i class="fas fa-rss"></i> <a type="application/rss+xml" href="/feed.xml">Subscribe</a></div>
+<div class="article__subscribe"><div class="subscribe"><i class="fas fa-rss"></i> <a type="application/rss+xml" href="/proxy/4000/feed.xml">Subscribe</a></div>
 </div><div class="article__license"></div></footer>
-<div class="article__section-navigator clearfix"><div class="previous"><span>PREVIOUS</span><a href="/posts/generate-word-cloud-with-chinese-fenci">Generate Word Cloud Figures with Chinese-Tokenization and WordCloud python libraries</a></div><div class="next"><span>NEXT</span><a href="/posts/eddl-how-do-we-train-on-limited-edge-devices-part2">EDDL: How do we train neural networks on limited edge devices - PART 2</a></div></div></div>
+<div class="article__section-navigator clearfix"><div class="previous"><span>PREVIOUS</span><a href="/proxy/4000/posts/generate-word-cloud-with-chinese-fenci">Generate Word Cloud Figures with Chinese-Tokenization and WordCloud python libraries</a></div><div class="next"><span>NEXT</span><a href="/proxy/4000/posts/eddl-how-do-we-train-on-limited-edge-devices-part2">EDDL: How do we train neural networks on limited edge devices - PART 2</a></div></div></div>
 
 </div>
 
@@ -569,7 +569,7 @@ Smartly schedule work balance and handle join/exit issues also need under consid
             </div></div></div><div class="page__footer d-print-none">
 <footer class="footer py-4 js-page-footer">
   <div class="main"><div itemscope itemtype="http://schema.org/Person">
-      <meta itemprop="name" content="Pengzhan Hao"><meta itemprop="url" content="/"><div class="footer__author-links"><div class="author-links">
+      <meta itemprop="name" content="Pengzhan Hao"><meta itemprop="url" content="/proxy/4000/"><div class="footer__author-links"><div class="author-links">
   <ul class="menu menu--nowrap menu--inline"><li title="Send me an Email.">
       <a class="button button--circle mail-button" itemprop="email" href="mailto:haopengzhan@gmail.com" target="_blank">
         <div class="icon"><svg fill="#000000" width="24px" height="24px" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg">
diff --git a/_site/posts/generate-word-cloud-with-chinese-fenci.html b/_site/posts/generate-word-cloud-with-chinese-fenci.html
index 626cd6b..af3f408 100644
--- a/_site/posts/generate-word-cloud-with-chinese-fenci.html
+++ b/_site/posts/generate-word-cloud-with-chinese-fenci.html
@@ -4,7 +4,7 @@
 <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no"><title>Generate Word Cloud Figures with Chinese-Tokenization and WordCloud python libraries - STSD</title>
 
 <meta name="description" content="Let’s generate a word cloud like this. Don’t understand the language is not a big deal.If your written language is based on latin alphabet(or other language ...">
-<link rel="canonical" href="https://blog.pengzhan.dev/posts/generate-word-cloud-with-chinese-fenci"><link rel="alternate" type="application/rss+xml" title="STSD" href="/feed.xml"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
+<link rel="canonical" href="https://dev.pengzhan.dev/proxy/4000/posts/generate-word-cloud-with-chinese-fenci"><link rel="alternate" type="application/rss+xml" title="STSD" href="/proxy/4000/feed.xml"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
 <link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png">
 <link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png">
 <link rel="manifest" href="/assets/site.webmanifest">
@@ -13,7 +13,7 @@
 <meta name="msapplication-TileColor" content="#da532c">
 <meta name="msapplication-config" content="/assets/browserconfig.xml">
 <meta name="theme-color" content="#ffffff">
-<link rel="stylesheet" href="/assets/css/main.css"><link rel="stylesheet" href="/static/2022-09/all.css" >
+<link rel="stylesheet" href="/proxy/4000/assets/css/main.css"><link rel="stylesheet" href="/static/2022-09/all.css" >
 
 <style>
     @import url('https://fonts.googleapis.com/css2?family=Poppins:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;0,800;0,900;1,100;1,200;1,300;1,400;1,500;1,600;1,700;1,800;1,900&family=Roboto+Slab:wght@100..900&family=ZCOOL+XiaoWei&display=swap')
@@ -26,15 +26,15 @@
 <meta property="og:locale" content="en" />
 <meta name="description" content="Let’s generate a word cloud like this. Don’t understand the language is not a big deal. If your written language is based on latin alphabet(or other language has space between words), skip tokenization." />
 <meta property="og:description" content="Let’s generate a word cloud like this. Don’t understand the language is not a big deal. If your written language is based on latin alphabet(or other language has space between words), skip tokenization." />
-<link rel="canonical" href="https://blog.pengzhan.dev/posts/generate-word-cloud-with-chinese-fenci" />
-<meta property="og:url" content="https://blog.pengzhan.dev/posts/generate-word-cloud-with-chinese-fenci" />
+<link rel="canonical" href="https://dev.pengzhan.dev/posts/generate-word-cloud-with-chinese-fenci" />
+<meta property="og:url" content="https://dev.pengzhan.dev/posts/generate-word-cloud-with-chinese-fenci" />
 <meta property="og:site_name" content="STSD" />
 <meta property="og:type" content="article" />
 <meta property="article:published_time" content="2020-09-15T22:00:14-04:00" />
 <meta name="twitter:card" content="summary" />
 <meta property="twitter:title" content="Generate Word Cloud Figures with Chinese-Tokenization and WordCloud python libraries" />
 <script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"Pengzhan Hao"},"dateModified":"2021-12-11T13:35:39-05:00","datePublished":"2020-09-15T22:00:14-04:00","description":"Let’s generate a word cloud like this. Don’t understand the language is not a big deal. If your written language is based on latin alphabet(or other language has space between words), skip tokenization.","headline":"Generate Word Cloud Figures with Chinese-Tokenization and WordCloud python libraries","mainEntityOfPage":{"@type":"WebPage","@id":"https://blog.pengzhan.dev/posts/generate-word-cloud-with-chinese-fenci"},"url":"https://blog.pengzhan.dev/posts/generate-word-cloud-with-chinese-fenci"}</script>
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"Pengzhan Hao"},"dateModified":"2021-12-11T13:35:39-05:00","datePublished":"2020-09-15T22:00:14-04:00","description":"Let’s generate a word cloud like this. Don’t understand the language is not a big deal. If your written language is based on latin alphabet(or other language has space between words), skip tokenization.","headline":"Generate Word Cloud Figures with Chinese-Tokenization and WordCloud python libraries","mainEntityOfPage":{"@type":"WebPage","@id":"https://dev.pengzhan.dev/posts/generate-word-cloud-with-chinese-fenci"},"url":"https://dev.pengzhan.dev/posts/generate-word-cloud-with-chinese-fenci"}</script>
 <!-- End Jekyll SEO tag -->
 
 <!-- end custom head snippets -->
@@ -271,7 +271,7 @@
         }
       },
       paths: {
-        search_js: '/assets/search.js'
+        search_js: '/proxy/4000/assets/search.js'
       }
     };
     window.TEXT_VARIABLES = TEXT_VARIABLES;
@@ -426,9 +426,9 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
 </g>
 </svg>
 -->
-          <img src="/assets/favicon-32x32.png" style="margin-right: 5px;"><a title="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
-" href="/">STSD</a></div><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></div><nav class="navigation">
-        <ul><li class="navigation__item"><a href="/archive.html">Archive</a></li><li class="navigation__item"><a href="https://pengzhan.dev">About</a></li><li><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></li></ul>
+          <img src="/proxy/4000/assets/favicon-32x32.png" style="margin-right: 5px;"><a title="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
+" href="/proxy/4000/">STSD</a></div><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></div><nav class="navigation">
+        <ul><li class="navigation__item"><a href="/proxy/4000/archive.html">Archive</a></li><li class="navigation__item"><a href="https://pengzhan.dev">About</a></li><li><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></li></ul>
       </nav></div>
   </header>
 </div><div class="page__content"><div class ="main"><div class="grid grid--reverse">
@@ -441,7 +441,7 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
 <!-- end custom main top snippet -->
 <article itemscope itemtype="http://schema.org/Article"><div class="article__header"><header><h1>Generate Word Cloud Figures with Chinese-Tokenization and WordCloud python libraries</h1></header></div><meta itemprop="headline" content="Generate Word Cloud Figures with Chinese-Tokenization and WordCloud python libraries"><div class="article__info clearfix"><ul class="left-col menu"><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Visualization">Visualization</a>
+                href="/proxy/4000/archive.html?tag=Visualization">Visualization</a>
             </li></ul><ul class="right-col menu"><li><i class="fas fa-user"></i> <span>Pengzhan Hao</span></li><li><i class="far fa-calendar-alt"></i> <span>Sep 15, 2020</span>
             </li></ul></div><meta itemprop="author" content="Pengzhan Hao"/><meta itemprop="datePublished" content="2020-09-15T22:00:14-04:00">
     <meta itemprop="keywords" content="Visualization"><div class="js-article-content"><div class="layout--article"><!-- start custom article top snippet -->
@@ -611,9 +611,9 @@ If your written language is based on latin alphabet(or other language has space
 </div><section class="article__sharing d-print-none"></section><div class="d-print-none"><footer class="article__footer"><meta itemprop="dateModified" content="2020-09-15T22:00:14-04:00"><!-- start custom article footer snippet -->
 
 <!-- end custom article footer snippet -->
-<div class="article__subscribe"><div class="subscribe"><i class="fas fa-rss"></i> <a type="application/rss+xml" href="/feed.xml">Subscribe</a></div>
+<div class="article__subscribe"><div class="subscribe"><i class="fas fa-rss"></i> <a type="application/rss+xml" href="/proxy/4000/feed.xml">Subscribe</a></div>
 </div><div class="article__license"></div></footer>
-<div class="article__section-navigator clearfix"><div class="previous"><span>PREVIOUS</span><a href="/posts/intro-xv6">Xv6 introduction</a></div><div class="next"><span>NEXT</span><a href="/posts/eddl-how-do-we-train-on-limited-edge-devices">EDDL: How do we train neural networks on limited edge devices - PART 1</a></div></div></div>
+<div class="article__section-navigator clearfix"><div class="previous"><span>PREVIOUS</span><a href="/proxy/4000/posts/intro-xv6">Xv6 introduction</a></div><div class="next"><span>NEXT</span><a href="/proxy/4000/posts/eddl-how-do-we-train-on-limited-edge-devices">EDDL: How do we train neural networks on limited edge devices - PART 1</a></div></div></div>
 
 </div>
 
@@ -649,7 +649,7 @@ If your written language is based on latin alphabet(or other language has space
             </div></div></div><div class="page__footer d-print-none">
 <footer class="footer py-4 js-page-footer">
   <div class="main"><div itemscope itemtype="http://schema.org/Person">
-      <meta itemprop="name" content="Pengzhan Hao"><meta itemprop="url" content="/"><div class="footer__author-links"><div class="author-links">
+      <meta itemprop="name" content="Pengzhan Hao"><meta itemprop="url" content="/proxy/4000/"><div class="footer__author-links"><div class="author-links">
   <ul class="menu menu--nowrap menu--inline"><li title="Send me an Email.">
       <a class="button button--circle mail-button" itemprop="email" href="mailto:haopengzhan@gmail.com" target="_blank">
         <div class="icon"><svg fill="#000000" width="24px" height="24px" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg">
diff --git a/_site/posts/intro-xv6.html b/_site/posts/intro-xv6.html
index e986be2..2ac9c37 100644
--- a/_site/posts/intro-xv6.html
+++ b/_site/posts/intro-xv6.html
@@ -4,7 +4,7 @@
 <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no"><title>Xv6 introduction - STSD</title>
 
 <meta name="description" content="In this post, you will learn a few basic concepts of xv6. Learning path will be closed coupled to first project assignment I gave when I assisted in teaching...">
-<link rel="canonical" href="https://blog.pengzhan.dev/posts/intro-xv6"><link rel="alternate" type="application/rss+xml" title="STSD" href="/feed.xml"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
+<link rel="canonical" href="https://dev.pengzhan.dev/proxy/4000/posts/intro-xv6"><link rel="alternate" type="application/rss+xml" title="STSD" href="/proxy/4000/feed.xml"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
 <link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png">
 <link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png">
 <link rel="manifest" href="/assets/site.webmanifest">
@@ -13,7 +13,7 @@
 <meta name="msapplication-TileColor" content="#da532c">
 <meta name="msapplication-config" content="/assets/browserconfig.xml">
 <meta name="theme-color" content="#ffffff">
-<link rel="stylesheet" href="/assets/css/main.css"><link rel="stylesheet" href="/static/2022-09/all.css" >
+<link rel="stylesheet" href="/proxy/4000/assets/css/main.css"><link rel="stylesheet" href="/static/2022-09/all.css" >
 
 <style>
     @import url('https://fonts.googleapis.com/css2?family=Poppins:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;0,800;0,900;1,100;1,200;1,300;1,400;1,500;1,600;1,700;1,800;1,900&family=Roboto+Slab:wght@100..900&family=ZCOOL+XiaoWei&display=swap')
@@ -26,15 +26,15 @@
 <meta property="og:locale" content="en" />
 <meta name="description" content="In this post, you will learn a few basic concepts of xv6. Learning path will be closed coupled to first project assignment I gave when I assisted in teaching OS classes. Understand system call and know how to implement a simple one will be coved as the first half. In the second half of this post, I will discuss a little bit more on how to debug xv6 using gdb." />
 <meta property="og:description" content="In this post, you will learn a few basic concepts of xv6. Learning path will be closed coupled to first project assignment I gave when I assisted in teaching OS classes. Understand system call and know how to implement a simple one will be coved as the first half. In the second half of this post, I will discuss a little bit more on how to debug xv6 using gdb." />
-<link rel="canonical" href="https://blog.pengzhan.dev/posts/intro-xv6" />
-<meta property="og:url" content="https://blog.pengzhan.dev/posts/intro-xv6" />
+<link rel="canonical" href="https://dev.pengzhan.dev/posts/intro-xv6" />
+<meta property="og:url" content="https://dev.pengzhan.dev/posts/intro-xv6" />
 <meta property="og:site_name" content="STSD" />
 <meta property="og:type" content="article" />
 <meta property="article:published_time" content="2017-07-28T14:56:55-04:00" />
 <meta name="twitter:card" content="summary" />
 <meta property="twitter:title" content="Xv6 introduction" />
 <script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"Pengzhan Hao"},"dateModified":"2021-12-11T13:35:39-05:00","datePublished":"2017-07-28T14:56:55-04:00","description":"In this post, you will learn a few basic concepts of xv6. Learning path will be closed coupled to first project assignment I gave when I assisted in teaching OS classes. Understand system call and know how to implement a simple one will be coved as the first half. In the second half of this post, I will discuss a little bit more on how to debug xv6 using gdb.","headline":"Xv6 introduction","mainEntityOfPage":{"@type":"WebPage","@id":"https://blog.pengzhan.dev/posts/intro-xv6"},"url":"https://blog.pengzhan.dev/posts/intro-xv6"}</script>
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"Pengzhan Hao"},"dateModified":"2021-12-11T13:35:39-05:00","datePublished":"2017-07-28T14:56:55-04:00","description":"In this post, you will learn a few basic concepts of xv6. Learning path will be closed coupled to first project assignment I gave when I assisted in teaching OS classes. Understand system call and know how to implement a simple one will be coved as the first half. In the second half of this post, I will discuss a little bit more on how to debug xv6 using gdb.","headline":"Xv6 introduction","mainEntityOfPage":{"@type":"WebPage","@id":"https://dev.pengzhan.dev/posts/intro-xv6"},"url":"https://dev.pengzhan.dev/posts/intro-xv6"}</script>
 <!-- End Jekyll SEO tag -->
 
 <!-- end custom head snippets -->
@@ -271,7 +271,7 @@
         }
       },
       paths: {
-        search_js: '/assets/search.js'
+        search_js: '/proxy/4000/assets/search.js'
       }
     };
     window.TEXT_VARIABLES = TEXT_VARIABLES;
@@ -426,9 +426,9 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
 </g>
 </svg>
 -->
-          <img src="/assets/favicon-32x32.png" style="margin-right: 5px;"><a title="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
-" href="/">STSD</a></div><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></div><nav class="navigation">
-        <ul><li class="navigation__item"><a href="/archive.html">Archive</a></li><li class="navigation__item"><a href="https://pengzhan.dev">About</a></li><li><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></li></ul>
+          <img src="/proxy/4000/assets/favicon-32x32.png" style="margin-right: 5px;"><a title="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
+" href="/proxy/4000/">STSD</a></div><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></div><nav class="navigation">
+        <ul><li class="navigation__item"><a href="/proxy/4000/archive.html">Archive</a></li><li class="navigation__item"><a href="https://pengzhan.dev">About</a></li><li><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></li></ul>
       </nav></div>
   </header>
 </div><div class="page__content"><div class ="main"><div class="grid grid--reverse">
@@ -441,13 +441,13 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
 <!-- end custom main top snippet -->
 <article itemscope itemtype="http://schema.org/Article"><div class="article__header"><header><h1>Xv6 introduction</h1></header></div><meta itemprop="headline" content="Xv6 introduction"><div class="article__info clearfix"><ul class="left-col menu"><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Xv6">Xv6</a>
+                href="/proxy/4000/archive.html?tag=Xv6">Xv6</a>
             </li><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Teaching">Teaching</a>
+                href="/proxy/4000/archive.html?tag=Teaching">Teaching</a>
             </li><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Operating+system">Operating system</a>
+                href="/proxy/4000/archive.html?tag=Operating+system">Operating system</a>
             </li></ul><ul class="right-col menu"><li><i class="fas fa-user"></i> <span>Pengzhan Hao</span></li><li><i class="far fa-calendar-alt"></i> <span>Jul 28, 2017</span>
             </li></ul></div><meta itemprop="author" content="Pengzhan Hao"/><meta itemprop="datePublished" content="2017-07-28T14:56:55-04:00">
     <meta itemprop="keywords" content="Xv6,Teaching,Operating system"><div class="js-article-content"><div class="layout--article"><!-- start custom article top snippet -->
@@ -532,9 +532,9 @@ Using ssh may connect to different physical devices under same domain name, this
 </div><section class="article__sharing d-print-none"></section><div class="d-print-none"><footer class="article__footer"><meta itemprop="dateModified" content="2017-07-28T14:56:55-04:00"><!-- start custom article footer snippet -->
 
 <!-- end custom article footer snippet -->
-<div class="article__subscribe"><div class="subscribe"><i class="fas fa-rss"></i> <a type="application/rss+xml" href="/feed.xml">Subscribe</a></div>
+<div class="article__subscribe"><div class="subscribe"><i class="fas fa-rss"></i> <a type="application/rss+xml" href="/proxy/4000/feed.xml">Subscribe</a></div>
 </div><div class="article__license"></div></footer>
-<div class="article__section-navigator clearfix"><div class="previous"><span>PREVIOUS</span><a href="/posts/some-of-my-previews-exper-work">Some of my previews experiment works: 2016</a></div><div class="next"><span>NEXT</span><a href="/posts/generate-word-cloud-with-chinese-fenci">Generate Word Cloud Figures with Chinese-Tokenization and WordCloud python libraries</a></div></div></div>
+<div class="article__section-navigator clearfix"><div class="previous"><span>PREVIOUS</span><a href="/proxy/4000/posts/some-of-my-previews-exper-work">Some of my previews experiment works: 2016</a></div><div class="next"><span>NEXT</span><a href="/proxy/4000/posts/generate-word-cloud-with-chinese-fenci">Generate Word Cloud Figures with Chinese-Tokenization and WordCloud python libraries</a></div></div></div>
 
 </div>
 
@@ -570,7 +570,7 @@ Using ssh may connect to different physical devices under same domain name, this
             </div></div></div><div class="page__footer d-print-none">
 <footer class="footer py-4 js-page-footer">
   <div class="main"><div itemscope itemtype="http://schema.org/Person">
-      <meta itemprop="name" content="Pengzhan Hao"><meta itemprop="url" content="/"><div class="footer__author-links"><div class="author-links">
+      <meta itemprop="name" content="Pengzhan Hao"><meta itemprop="url" content="/proxy/4000/"><div class="footer__author-links"><div class="author-links">
   <ul class="menu menu--nowrap menu--inline"><li title="Send me an Email.">
       <a class="button button--circle mail-button" itemprop="email" href="mailto:haopengzhan@gmail.com" target="_blank">
         <div class="icon"><svg fill="#000000" width="24px" height="24px" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg">
diff --git a/_site/posts/some-of-my-previews-exper-work.html b/_site/posts/some-of-my-previews-exper-work.html
index 20bd818..1d04b8c 100644
--- a/_site/posts/some-of-my-previews-exper-work.html
+++ b/_site/posts/some-of-my-previews-exper-work.html
@@ -4,7 +4,7 @@
 <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no"><title>Some of my previews experiment works: 2016 - STSD</title>
 
 <meta name="description" content="This blog contains only some basic record of my works. For some details, I will write a unique blog just for some specific topics.">
-<link rel="canonical" href="https://blog.pengzhan.dev/posts/some-of-my-previews-exper-work"><link rel="alternate" type="application/rss+xml" title="STSD" href="/feed.xml"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
+<link rel="canonical" href="https://dev.pengzhan.dev/proxy/4000/posts/some-of-my-previews-exper-work"><link rel="alternate" type="application/rss+xml" title="STSD" href="/proxy/4000/feed.xml"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
 <link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png">
 <link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png">
 <link rel="manifest" href="/assets/site.webmanifest">
@@ -13,7 +13,7 @@
 <meta name="msapplication-TileColor" content="#da532c">
 <meta name="msapplication-config" content="/assets/browserconfig.xml">
 <meta name="theme-color" content="#ffffff">
-<link rel="stylesheet" href="/assets/css/main.css"><link rel="stylesheet" href="/static/2022-09/all.css" >
+<link rel="stylesheet" href="/proxy/4000/assets/css/main.css"><link rel="stylesheet" href="/static/2022-09/all.css" >
 
 <style>
     @import url('https://fonts.googleapis.com/css2?family=Poppins:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;0,800;0,900;1,100;1,200;1,300;1,400;1,500;1,600;1,700;1,800;1,900&family=Roboto+Slab:wght@100..900&family=ZCOOL+XiaoWei&display=swap')
@@ -26,15 +26,15 @@
 <meta property="og:locale" content="en" />
 <meta name="description" content="This blog contains only some basic record of my works. For some details, I will write a unique blog just for some specific topics." />
 <meta property="og:description" content="This blog contains only some basic record of my works. For some details, I will write a unique blog just for some specific topics." />
-<link rel="canonical" href="https://blog.pengzhan.dev/posts/some-of-my-previews-exper-work" />
-<meta property="og:url" content="https://blog.pengzhan.dev/posts/some-of-my-previews-exper-work" />
+<link rel="canonical" href="https://dev.pengzhan.dev/posts/some-of-my-previews-exper-work" />
+<meta property="og:url" content="https://dev.pengzhan.dev/posts/some-of-my-previews-exper-work" />
 <meta property="og:site_name" content="STSD" />
 <meta property="og:type" content="article" />
 <meta property="article:published_time" content="2016-10-28T12:27:33-04:00" />
 <meta name="twitter:card" content="summary" />
 <meta property="twitter:title" content="Some of my previews experiment works: 2016" />
 <script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"Pengzhan Hao"},"dateModified":"2021-12-11T13:35:39-05:00","datePublished":"2016-10-28T12:27:33-04:00","description":"This blog contains only some basic record of my works. For some details, I will write a unique blog just for some specific topics.","headline":"Some of my previews experiment works: 2016","mainEntityOfPage":{"@type":"WebPage","@id":"https://blog.pengzhan.dev/posts/some-of-my-previews-exper-work"},"url":"https://blog.pengzhan.dev/posts/some-of-my-previews-exper-work"}</script>
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"Pengzhan Hao"},"dateModified":"2021-12-11T13:35:39-05:00","datePublished":"2016-10-28T12:27:33-04:00","description":"This blog contains only some basic record of my works. For some details, I will write a unique blog just for some specific topics.","headline":"Some of my previews experiment works: 2016","mainEntityOfPage":{"@type":"WebPage","@id":"https://dev.pengzhan.dev/posts/some-of-my-previews-exper-work"},"url":"https://dev.pengzhan.dev/posts/some-of-my-previews-exper-work"}</script>
 <!-- End Jekyll SEO tag -->
 
 <!-- end custom head snippets -->
@@ -271,7 +271,7 @@
         }
       },
       paths: {
-        search_js: '/assets/search.js'
+        search_js: '/proxy/4000/assets/search.js'
       }
     };
     window.TEXT_VARIABLES = TEXT_VARIABLES;
@@ -426,9 +426,9 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
 </g>
 </svg>
 -->
-          <img src="/assets/favicon-32x32.png" style="margin-right: 5px;"><a title="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
-" href="/">STSD</a></div><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></div><nav class="navigation">
-        <ul><li class="navigation__item"><a href="/archive.html">Archive</a></li><li class="navigation__item"><a href="https://pengzhan.dev">About</a></li><li><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></li></ul>
+          <img src="/proxy/4000/assets/favicon-32x32.png" style="margin-right: 5px;"><a title="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
+" href="/proxy/4000/">STSD</a></div><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></div><nav class="navigation">
+        <ul><li class="navigation__item"><a href="/proxy/4000/archive.html">Archive</a></li><li class="navigation__item"><a href="https://pengzhan.dev">About</a></li><li><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></li></ul>
       </nav></div>
   </header>
 </div><div class="page__content"><div class ="main"><div class="grid grid--reverse">
@@ -441,13 +441,13 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
 <!-- end custom main top snippet -->
 <article itemscope itemtype="http://schema.org/Article"><div class="article__header"><header><h1>Some of my previews experiment works: 2016</h1></header></div><meta itemprop="headline" content="Some of my previews experiment works: 2016"><div class="article__info clearfix"><ul class="left-col menu"><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Research">Research</a>
+                href="/proxy/4000/archive.html?tag=Research">Research</a>
             </li><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Log">Log</a>
+                href="/proxy/4000/archive.html?tag=Log">Log</a>
             </li><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Miscellanies">Miscellanies</a>
+                href="/proxy/4000/archive.html?tag=Miscellanies">Miscellanies</a>
             </li></ul><ul class="right-col menu"><li><i class="fas fa-user"></i> <span>Pengzhan Hao</span></li><li><i class="far fa-calendar-alt"></i> <span>Oct 28, 2016</span>
             </li></ul></div><meta itemprop="author" content="Pengzhan Hao"/><meta itemprop="datePublished" content="2016-10-28T12:27:33-04:00">
     <meta itemprop="keywords" content="Research,Log,Miscellanies"><div class="js-article-content"><div class="layout--article"><!-- start custom article top snippet -->
@@ -567,9 +567,9 @@ su
 </div><section class="article__sharing d-print-none"></section><div class="d-print-none"><footer class="article__footer"><meta itemprop="dateModified" content="2016-10-28T12:27:33-04:00"><!-- start custom article footer snippet -->
 
 <!-- end custom article footer snippet -->
-<div class="article__subscribe"><div class="subscribe"><i class="fas fa-rss"></i> <a type="application/rss+xml" href="/feed.xml">Subscribe</a></div>
+<div class="article__subscribe"><div class="subscribe"><i class="fas fa-rss"></i> <a type="application/rss+xml" href="/proxy/4000/feed.xml">Subscribe</a></div>
 </div><div class="article__license"></div></footer>
-<div class="article__section-navigator clearfix"><div class="previous"><span>PREVIOUS</span><a href="/posts/charles-is-not-a-good-tool">Using charles proxy to monitor mobile SSL traffics</a></div><div class="next"><span>NEXT</span><a href="/posts/intro-xv6">Xv6 introduction</a></div></div></div>
+<div class="article__section-navigator clearfix"><div class="previous"><span>PREVIOUS</span><a href="/proxy/4000/posts/charles-is-not-a-good-tool">Using charles proxy to monitor mobile SSL traffics</a></div><div class="next"><span>NEXT</span><a href="/proxy/4000/posts/intro-xv6">Xv6 introduction</a></div></div></div>
 
 </div>
 
@@ -605,7 +605,7 @@ su
             </div></div></div><div class="page__footer d-print-none">
 <footer class="footer py-4 js-page-footer">
   <div class="main"><div itemscope itemtype="http://schema.org/Person">
-      <meta itemprop="name" content="Pengzhan Hao"><meta itemprop="url" content="/"><div class="footer__author-links"><div class="author-links">
+      <meta itemprop="name" content="Pengzhan Hao"><meta itemprop="url" content="/proxy/4000/"><div class="footer__author-links"><div class="author-links">
   <ul class="menu menu--nowrap menu--inline"><li title="Send me an Email.">
       <a class="button button--circle mail-button" itemprop="email" href="mailto:haopengzhan@gmail.com" target="_blank">
         <div class="icon"><svg fill="#000000" width="24px" height="24px" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg">
diff --git a/_site/posts/welcome-to-my-blog.html b/_site/posts/welcome-to-my-blog.html
index b7ee9b4..455da88 100644
--- a/_site/posts/welcome-to-my-blog.html
+++ b/_site/posts/welcome-to-my-blog.html
@@ -4,7 +4,7 @@
 <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no"><title>STSD: Stop Talking Start Doing - STSD</title>
 
 <meta name="description" content="">
-<link rel="canonical" href="https://blog.pengzhan.dev/posts/welcome-to-my-blog"><link rel="alternate" type="application/rss+xml" title="STSD" href="/feed.xml"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
+<link rel="canonical" href="https://dev.pengzhan.dev/proxy/4000/posts/welcome-to-my-blog"><link rel="alternate" type="application/rss+xml" title="STSD" href="/proxy/4000/feed.xml"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
 <link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png">
 <link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png">
 <link rel="manifest" href="/assets/site.webmanifest">
@@ -13,7 +13,7 @@
 <meta name="msapplication-TileColor" content="#da532c">
 <meta name="msapplication-config" content="/assets/browserconfig.xml">
 <meta name="theme-color" content="#ffffff">
-<link rel="stylesheet" href="/assets/css/main.css"><link rel="stylesheet" href="/static/2022-09/all.css" >
+<link rel="stylesheet" href="/proxy/4000/assets/css/main.css"><link rel="stylesheet" href="/static/2022-09/all.css" >
 
 <style>
     @import url('https://fonts.googleapis.com/css2?family=Poppins:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;0,800;0,900;1,100;1,200;1,300;1,400;1,500;1,600;1,700;1,800;1,900&family=Roboto+Slab:wght@100..900&family=ZCOOL+XiaoWei&display=swap')
@@ -26,15 +26,15 @@
 <meta property="og:locale" content="en" />
 <meta name="description" content="My personal blog, some contents are useful, the others are not. Just like my mediocre life." />
 <meta property="og:description" content="My personal blog, some contents are useful, the others are not. Just like my mediocre life." />
-<link rel="canonical" href="https://blog.pengzhan.dev/posts/welcome-to-my-blog" />
-<meta property="og:url" content="https://blog.pengzhan.dev/posts/welcome-to-my-blog" />
+<link rel="canonical" href="https://dev.pengzhan.dev/posts/welcome-to-my-blog" />
+<meta property="og:url" content="https://dev.pengzhan.dev/posts/welcome-to-my-blog" />
 <meta property="og:site_name" content="STSD" />
 <meta property="og:type" content="article" />
 <meta property="article:published_time" content="2016-10-26T22:50:33-04:00" />
 <meta name="twitter:card" content="summary" />
 <meta property="twitter:title" content="STSD: Stop Talking Start Doing" />
 <script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"Pengzhan Hao"},"dateModified":"2024-04-03T18:23:31-04:00","datePublished":"2016-10-26T22:50:33-04:00","description":"My personal blog, some contents are useful, the others are not. Just like my mediocre life.","headline":"STSD: Stop Talking Start Doing","mainEntityOfPage":{"@type":"WebPage","@id":"https://blog.pengzhan.dev/posts/welcome-to-my-blog"},"url":"https://blog.pengzhan.dev/posts/welcome-to-my-blog"}</script>
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"Pengzhan Hao"},"dateModified":"2024-04-03T18:23:31-04:00","datePublished":"2016-10-26T22:50:33-04:00","description":"My personal blog, some contents are useful, the others are not. Just like my mediocre life.","headline":"STSD: Stop Talking Start Doing","mainEntityOfPage":{"@type":"WebPage","@id":"https://dev.pengzhan.dev/posts/welcome-to-my-blog"},"url":"https://dev.pengzhan.dev/posts/welcome-to-my-blog"}</script>
 <!-- End Jekyll SEO tag -->
 
 <!-- end custom head snippets -->
@@ -271,7 +271,7 @@
         }
       },
       paths: {
-        search_js: '/assets/search.js'
+        search_js: '/proxy/4000/assets/search.js'
       }
     };
     window.TEXT_VARIABLES = TEXT_VARIABLES;
@@ -426,9 +426,9 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
 </g>
 </svg>
 -->
-          <img src="/assets/favicon-32x32.png" style="margin-right: 5px;"><a title="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
-" href="/">STSD</a></div><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></div><nav class="navigation">
-        <ul><li class="navigation__item"><a href="/archive.html">Archive</a></li><li class="navigation__item"><a href="https://pengzhan.dev">About</a></li><li><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></li></ul>
+          <img src="/proxy/4000/assets/favicon-32x32.png" style="margin-right: 5px;"><a title="My personal blog, some contents are useful, the others are not. Just like my mediocre life.
+" href="/proxy/4000/">STSD</a></div><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></div><nav class="navigation">
+        <ul><li class="navigation__item"><a href="/proxy/4000/archive.html">Archive</a></li><li class="navigation__item"><a href="https://pengzhan.dev">About</a></li><li><button class="button button--secondary button--circle search-button js-search-toggle"><i class="fas fa-search"></i></button></li></ul>
       </nav></div>
   </header>
 </div><div class="page__content"><div class ="main"><div class="grid grid--reverse">
@@ -441,7 +441,7 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
 <!-- end custom main top snippet -->
 <article itemscope itemtype="http://schema.org/Article"><div class="article__header"><header><h1>STSD: Stop Talking Start Doing</h1></header></div><meta itemprop="headline" content="STSD: Stop Talking Start Doing"><div class="article__info clearfix"><ul class="left-col menu"><li>
               <a class="button button--secondary button--pill button--sm"
-                href="/archive.html?tag=Diary">Diary</a>
+                href="/proxy/4000/archive.html?tag=Diary">Diary</a>
             </li></ul><ul class="right-col menu"><li><i class="far fa-calendar-alt"></i> <span>Oct 26, 2016</span>
             </li></ul></div><meta itemprop="author" content="Pengzhan Hao"/><meta itemprop="datePublished" content="2016-10-26T22:50:33-04:00">
     <meta itemprop="keywords" content="Diary"><div class="js-article-content"><div class="layout--article"><!-- start custom article top snippet -->
@@ -451,9 +451,9 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
 </div><section class="article__sharing d-print-none"></section><div class="d-print-none"><footer class="article__footer"><meta itemprop="dateModified" content="2016-10-26T22:50:33-04:00"><!-- start custom article footer snippet -->
 
 <!-- end custom article footer snippet -->
-<div class="article__subscribe"><div class="subscribe"><i class="fas fa-rss"></i> <a type="application/rss+xml" href="/feed.xml">Subscribe</a></div>
+<div class="article__subscribe"><div class="subscribe"><i class="fas fa-rss"></i> <a type="application/rss+xml" href="/proxy/4000/feed.xml">Subscribe</a></div>
 </div><div class="article__license"></div></footer>
-<div class="article__section-navigator clearfix"><div class="next"><span>NEXT</span><a href="/posts/charles-is-not-a-good-tool">Using charles proxy to monitor mobile SSL traffics</a></div></div></div>
+<div class="article__section-navigator clearfix"><div class="next"><span>NEXT</span><a href="/proxy/4000/posts/charles-is-not-a-good-tool">Using charles proxy to monitor mobile SSL traffics</a></div></div></div>
 
 </div>
 
@@ -489,7 +489,7 @@ c13 9 26 20 30 26 7 11 -9 26 -27 26 -5 0 -3 -5 5 -10 9 -6 10 -10 3 -10 -24
             </div></div></div><div class="page__footer d-print-none">
 <footer class="footer py-4 js-page-footer">
   <div class="main"><div itemscope itemtype="http://schema.org/Person">
-      <meta itemprop="name" content="Pengzhan Hao"><meta itemprop="url" content="/"><div class="footer__author-links"><div class="author-links">
+      <meta itemprop="name" content="Pengzhan Hao"><meta itemprop="url" content="/proxy/4000/"><div class="footer__author-links"><div class="author-links">
   <ul class="menu menu--nowrap menu--inline"><li title="Send me an Email.">
       <a class="button button--circle mail-button" itemprop="email" href="mailto:haopengzhan@gmail.com" target="_blank">
         <div class="icon"><svg fill="#000000" width="24px" height="24px" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg">
diff --git a/_site/robots.txt b/_site/robots.txt
index 6c9fc8d..f770bad 100644
--- a/_site/robots.txt
+++ b/_site/robots.txt
@@ -1 +1 @@
-Sitemap: https://blog.pengzhan.dev/sitemap.xml
+Sitemap: https://dev.pengzhan.dev/sitemap.xml
diff --git a/_site/sitemap.xml b/_site/sitemap.xml
index 58b43b0..7a61f79 100644
--- a/_site/sitemap.xml
+++ b/_site/sitemap.xml
@@ -1,75 +1,79 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd" xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
 <url>
-<loc>https://blog.pengzhan.dev/posts/welcome-to-my-blog</loc>
+<loc>https://dev.pengzhan.dev/posts/welcome-to-my-blog</loc>
 <lastmod>2024-04-03T18:23:31-04:00</lastmod>
 </url>
 <url>
-<loc>https://blog.pengzhan.dev/posts/charles-is-not-a-good-tool</loc>
+<loc>https://dev.pengzhan.dev/posts/charles-is-not-a-good-tool</loc>
 <lastmod>2021-12-11T13:35:39-05:00</lastmod>
 </url>
 <url>
-<loc>https://blog.pengzhan.dev/posts/some-of-my-previews-exper-work</loc>
+<loc>https://dev.pengzhan.dev/posts/some-of-my-previews-exper-work</loc>
 <lastmod>2021-12-11T13:35:39-05:00</lastmod>
 </url>
 <url>
-<loc>https://blog.pengzhan.dev/posts/intro-xv6</loc>
+<loc>https://dev.pengzhan.dev/posts/intro-xv6</loc>
 <lastmod>2021-12-11T13:35:39-05:00</lastmod>
 </url>
 <url>
-<loc>https://blog.pengzhan.dev/posts/generate-word-cloud-with-chinese-fenci</loc>
+<loc>https://dev.pengzhan.dev/posts/generate-word-cloud-with-chinese-fenci</loc>
 <lastmod>2021-12-11T13:35:39-05:00</lastmod>
 </url>
 <url>
-<loc>https://blog.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices</loc>
+<loc>https://dev.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices</loc>
 <lastmod>2021-12-11T13:35:39-05:00</lastmod>
 </url>
 <url>
-<loc>https://blog.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices-part2</loc>
+<loc>https://dev.pengzhan.dev/posts/eddl-how-do-we-train-on-limited-edge-devices-part2</loc>
 <lastmod>2021-12-11T13:35:39-05:00</lastmod>
 </url>
 <url>
-<loc>https://blog.pengzhan.dev/posts/cs350-labs</loc>
+<loc>https://dev.pengzhan.dev/posts/cs350-labs</loc>
 <lastmod>2022-05-04T19:45:56-04:00</lastmod>
 </url>
 <url>
-<loc>https://blog.pengzhan.dev/posts/Debug-kubelet</loc>
+<loc>https://dev.pengzhan.dev/posts/Debug-kubelet</loc>
 <lastmod>2024-04-10T23:51:19-04:00</lastmod>
 </url>
 <url>
-<loc>https://blog.pengzhan.dev/archive</loc>
+<loc>https://dev.pengzhan.dev/posts/inotify-watcher-leaks-in-kubelet</loc>
+<lastmod>2024-04-11T16:57:07-04:00</lastmod>
+</url>
+<url>
+<loc>https://dev.pengzhan.dev/archive</loc>
 <lastmod>2021-10-25T14:53:15-04:00</lastmod>
 </url>
 <url>
-<loc>https://blog.pengzhan.dev/</loc>
+<loc>https://dev.pengzhan.dev/</loc>
 <lastmod>2021-10-25T14:53:15-04:00</lastmod>
 </url>
 <url>
-<loc>https://blog.pengzhan.dev/page2/</loc>
+<loc>https://dev.pengzhan.dev/page2/</loc>
 <lastmod>2021-10-25T14:53:15-04:00</lastmod>
 </url>
 <url>
-<loc>https://blog.pengzhan.dev/static/2017-07/eps_poster.pdf</loc>
+<loc>https://dev.pengzhan.dev/static/2017-07/eps_poster.pdf</loc>
 <lastmod>2024-04-03T17:11:49-04:00</lastmod>
 </url>
 <url>
-<loc>https://blog.pengzhan.dev/static/2017-07/p163-hao.pdf</loc>
+<loc>https://dev.pengzhan.dev/static/2017-07/p163-hao.pdf</loc>
 <lastmod>2024-04-03T17:11:49-04:00</lastmod>
 </url>
 <url>
-<loc>https://blog.pengzhan.dev/static/2017-07/p169-zhang.pdf</loc>
+<loc>https://dev.pengzhan.dev/static/2017-07/p169-zhang.pdf</loc>
 <lastmod>2024-04-03T17:11:49-04:00</lastmod>
 </url>
 <url>
-<loc>https://blog.pengzhan.dev/static/2018-02/a7-hao.pdf</loc>
+<loc>https://dev.pengzhan.dev/static/2018-02/a7-hao.pdf</loc>
 <lastmod>2024-04-03T17:11:49-04:00</lastmod>
 </url>
 <url>
-<loc>https://blog.pengzhan.dev/static/2021-10/eta-infocom18.pdf</loc>
+<loc>https://dev.pengzhan.dev/static/2021-10/eta-infocom18.pdf</loc>
 <lastmod>2024-04-03T17:11:49-04:00</lastmod>
 </url>
 <url>
-<loc>https://blog.pengzhan.dev/static/2021-12/eddl-sec21.pdf</loc>
+<loc>https://dev.pengzhan.dev/static/2021-12/eddl-sec21.pdf</loc>
 <lastmod>2024-04-03T17:11:49-04:00</lastmod>
 </url>
 </urlset>
diff --git a/static/2024-04/kubelet_inotify_leak_logo.png b/static/2024-04/kubelet_inotify_leak_logo.png
new file mode 100644
index 0000000..a8eb9d4
Binary files /dev/null and b/static/2024-04/kubelet_inotify_leak_logo.png differ
