diff --git a/roles/generate-cert/tasks/main.yml b/roles/generate-cert/tasks/main.yml
index f4ccaa9..451f2dd 100644
--- a/roles/generate-cert/tasks/main.yml
+++ b/roles/generate-cert/tasks/main.yml
@@ -30,3 +30,13 @@
     group: "{{ group | default('root') }}"
     mode: "{{ key_mode | default('u=rw,g=,o=') }}"
   when: not key_file.stat.exists
+
+
+# Clean up
+- name: Remove the local cert key
+  become: false
+  file:
+    path: "/tmp/ansible_hacky_pki_{{ cname }}.key"
+    state: absent
+  delegate_to: localhost
+  when: not key_file.stat.exists