Merge pull request 'master' (#1) from master into aurore

Reviewed-on: #1

configs/config.j2

@@ -43,13 +43,16 @@ ip dns server-address priority {{ loop.index }} {{ d }}
 {%- endfor %}
 ip ssh filetransfer
 {%- for i6 in ipv6_managers.values() %}
-ipv6 authorized-managers {{ i6.ip }} {{ i6.subnet }} access manager
+ipv6 authorized-managers {{ i6.ip }} {{ i6.subnet }} access manager
 {%- endfor %}
 {%- if ra_guard_ports %}
 ipv6 ra-guard ports {{ ra_guard_ports }}
 {%- endif %}
 {%- for iface in interfaces %}
 interface {{ iface.number }}
+{%- if iface.flowcontrol %}
+   flow-control
+{%- endif %}
    name "{{ iface.name }}"
 {%- if iface.dhcp_trust %}
    dhcp-snooping trust
@@ -57,9 +60,6 @@ interface {{ iface.number }}
 {%- if iface.dhcpv6_trust %}
    dhcpv6-snooping trust
 {%- endif %}
-{%- if iface.flowcontrol %}
-   flow-control
-{% endif %}
 {%- if iface.arp_trust %}
    arp-protect trust
 {%- endif %}

configs/master.yml

@@ -20,3 +20,4 @@ radius_servers:
     secret: ploptotoswitch1
 snmp_user: re2o
 unauth_redirect: http://intranet.crans.org/users/initial_register
+radius_logoff: 3600

generate-conf.py

@@ -1,10 +1,10 @@
 import argparse
 import difflib
+import getpass
 import logging
 
-from pprint import pprint
-
 import colorlog
+import requests
 import termcolor
 import yaml
 
@@ -142,8 +142,17 @@ def gen_interfaces(switch_config):
     interfaces.sort(key=lambda x: x["number"])
     return interfaces, vlans, mac_based_ports, ra_guard_ports, dhcp_snooping_vlans
 
-def gen_conf(master_config, switch_config, old_config):
+def get_header(old_config):
     header = "\n".join(old_config.split("\n")[:2])
+    return header
+
+def conf_from_dict(config_dict):
+    with open("configs/config.j2", "r") as template_file:
+        template = Template(template_file.read())
+        configuration = template.render(config_dict)
+    return configuration
+
+def gen_conf(master_config, switch_config, header):
     interfaces, vlans, mac_based_ports, ra_guard_ports, dhcp_snooping_vlans = gen_interfaces(switch_config)
     config_dict = {
         "header": header,
@@ -164,13 +173,157 @@ def gen_conf(master_config, switch_config, old_config):
         "unauth_redirect": master_config.get("unauth_redirect"),
         "dhcp_snooping_vlans": syntax_from_range(dhcp_snooping_vlans, vlan_syntax=True),
     }
-    with open("configs/config.j2", "r") as template_file:
+    return conf_from_dict(config_dict)
-        template = Template(template_file.read())
-        configuration = template.render(config_dict)
-    return configuration
 
+def gen_conf_re2o(re2o_config, header):
+    mgmt_utils = re2o_config.get("switchs_management_utils")
+    ipv4_managers = dict()
+    for m in mgmt_utils.get("subnet"):
+        ipv4_managers[m.get("network")] = { "ip": m.get("network"), "subnet": m.get("netmask")}
+    ipv6_managers = dict()
+    # FUCK YOU ! subnet6 c'est pas une liste de subnets mais un seul subnet
+    m = mgmt_utils.get("subnet6")
+    ipv6_managers[m.get("network")] = { "ip": m.get("network"), "subnet": m.get("netmask")}
+    vlans = dict()
+    dhcp_snooping_vlans = list()
+    dhcpv6_snooping_vlans = list()
+    arp_protect = { "vlans": list() }
+    for vlan in re2o_config.get("vlans"):
+        vlan_id = vlan["vlan_id"]
+        vlans[vlan_id] = vlan
+        range_tagged = list()
+        range_untagged = list()
+        # on récupère les ports tag et untagged
+        for port in re2o_config.get("ports"):
+            port_profile = port["get_port_profile"]
+            for v in port_profile["vlan_tagged"]:
+                if v["vlan_id"] == vlan_id:
+                    range_tagged.append(port["port"])
+            # i n'y a qu'un seul vlan untagged
+            v = port_profile["vlan_untagged"]
+            if v is not None and v["vlan_id"] == vlan_id:
+                range_untagged.append(port["port"])
+        vlans[vlan_id]["tagged"] = syntax_from_range(range_tagged)
+        vlans[vlan_id]["untagged"] = syntax_from_range(range_untagged)
+        # on rajoute les ips sur les vlans où il y en a
+        for address, iface in re2o_config.get("interfaces_subnet", dict()).items():
+            # ouais y'a une autre liste là, don't ask
+            for i in iface:
+                if i["vlan_id"] == vlan_id:
+                    if vlans[vlan_id].get("ip") is None:
+                        vlans[vlan_id]["ip"] = dict()
+                    vlans[vlan_id]["ip"]["addr"] = address
+                    vlans[vlan_id]["ip"]["subnet"] = i["netmask"]
+        for address, iface in re2o_config.get("interfaces6_subnet", dict()).items():
+            if iface["vlan_id"] == vlan_id:
+                if vlans[vlan_id].get("ip") is None:
+                    vlans[vlan_id]["ip"] = dict()
+                vlans[vlan_id]["ip"]["addr6"] = address
+                vlans[vlan_id]["ip"]["subnet6"] = iface["netmask_cidr"]
+        # on ajoute les vlans qui ont besoin du dhcp snooping
+        if vlan["dhcp_snooping"]:
+            dhcp_snooping_vlans.append(vlan_id)
+        if vlan["dhcpv6_snooping"]:
+            dhcpv6_snooping_vlans.append(vlan_id)
+        if vlan["arp_protect"]:
+            arp_protect["vlans"].append(vlan_id)
+        vlans[vlan_id]["ipv6_mld"] = vlan["mld"]
+    # on récupère les informations intéressantes sur les ports
+    mac_based_ports = list()
+    ra_guard_ports = list()
+    interfaces = list()
+    loop_protect = { "ports": list()}
+    for port in re2o_config.get("ports"):
+        port_profile = port["get_port_profile"]
+        if port_profile["ra_guard"]:
+            ra_guard_ports.append(port["port"])
+        if port_profile["loop_protect"]:
+            loop_protect["ports"].append(port["port"])
+        iface = {
+            "name": port["pretty_name"],
+            "number": port["port"],
+            "dhcp_trust": not port_profile["dhcp_snooping"],
+            "dhcpv6_trust": not port_profile["dhcpv6_snooping"],
+            "flowcontrol": port_profile["flow_control"],
+            "arp_trust": not port_profile["arp_protect"],
+        }
+        if port_profile["radius_type"] == "MAC-radius":
+            mac_based_ports.append(port["port"])
+            iface["mac_based"] = True
+            iface["addr_limit"] = port_profile["mac_limit"]
+            iface["logoff"] = master_config.get("radius_logoff")
+        interfaces.append(iface)
+    loop_protect["ports"] = syntax_from_range(loop_protect["ports"])
+    arp_protect["vlans"] = syntax_from_range(arp_protect["vlans"], vlan_syntax=True)
+    interfaces.sort(key=lambda x: x["number"])
+    radius_key = re2o_config.get("get_radius_key_value")
+    radius_servers = [ {"ip": i, "secret": radius_key } for i in mgmt_utils["radius_servers"]["ipv4"] + mgmt_utils["radius_servers"]["ipv6"]]
+    config_dict = {
+       "header": header,
+       "location": re2o_config.get("switchbay").get("name"),
+       "hostname": re2o_config.get("short_name"),
+       "dhcp_servers": mgmt_utils.get("dhcp_servers").get("ipv4"),
+       "dhcpv6_servers": mgmt_utils.get("dhcp_servers").get("ipv6"),
+       "ipv4_managers": ipv4_managers,
+       "ipv6_managers": ipv6_managers,
+       "vlans": vlans,
+       "dhcp_snooping_vlans": syntax_from_range(dhcp_snooping_vlans, vlan_syntax=True),
+       "dhcpv6_snooping_vlans": syntax_from_range(dhcpv6_snooping_vlans, vlan_syntax=True),
+       "logging": mgmt_utils["log_servers"]["ipv4"] + mgmt_utils["log_servers"]["ipv6"],
+       "sntp": mgmt_utils["ntp_servers"]["ipv4"] + mgmt_utils["ntp_servers"]["ipv6"],
+       "dns": mgmt_utils["dns_recursive_servers"]["ipv4"] + mgmt_utils["dns_recursive_servers"]["ipv6"],
+       "radius_servers": radius_servers,
+       "snmp_user": master_config.get("snmp_user"),
+       "unauth_redirect": master_config.get("unauth_redirect"),
+       "mac_based_ports": syntax_from_range(mac_based_ports),
+       "ra_guard_ports": syntax_from_range(ra_guard_ports),
+       "loop_protect": loop_protect,
+       "arp_protect": arp_protect,
+       "interfaces": interfaces,
+    }
 
+    return conf_from_dict(config_dict)
 
+def connect_as_self(base_url, username=None):
+    """Get a re2o token for the current unix user """
+    if username is None:
+        username = getpass.getuser()
+    r = requests.post(
+            base_url + "token-auth",
+            data={
+                "username": username,
+                "password": getpass.getpass("Login to {} as {} with password :".format(base_url, username))
+            }
+    )
+    if r.status_code != 200:
+        logger.critical("Wrong login/password")
+        exit(1)
+    token = r.json().get("token")
+    return token
+
+def get_switch_from_results(results, switch_name):
+    for s in results:
+        if s.get("short_name") == switch_name:
+            return s
+    return None
+
+def get_switch_from_re2o(re2o_instance, switch_name, re2o_user):
+    base_url = "{}/api/".format(re2o_instance)
+    token = connect_as_self(base_url, re2o_user)
+    headers = {"Authorization": "Token " + token}
+    # On récupère la config du bon switch
+    r = requests.get(base_url + "switchs/ports-config", headers=headers)
+    sw_config = get_switch_from_results(r.json()["results"], switch_name)
+    while r.json().get("next") and s is None:
+        r = requests.get(r.json().get("next"), headers=headers)
+        sw_config = get_switch_from_results(r.json()["results"], switch_name)
+    # on récupère les infos globales pour dhcp, managers etc.
+    r = requests.get(base_url + "preferences/optionaltopologie", headers=headers)
+    sw_config.update(r.json())
+    # on récupère la liste des vlans
+    r = requests.get(base_url + "machines/vlan", headers=headers)
+    sw_config.update({"vlans": r.json().get("results")})
+    return sw_config
 
 if __name__ == "__main__":
     format_string = "%(asctime)s - %(levelname)s - %(message)s"
@@ -186,8 +339,18 @@ if __name__ == "__main__":
     parser.add_argument("-w", "--whole", action="store_true", help="Affiche la configuration en entier au lieu du diff")
     parser.add_argument("switch_name", type=str, help="Génère le template de ce switch")
     parser.add_argument("-H", "--host", type=str, required=False, help="Host sur lequel de déployer la configuration au lieu de l'adresse dans le template")
+    parser.add_argument("-r", "--re2o", type=str, required=False, help="Si renseigné, la configuration sera récupérée depuis l'instance de re2o indiquée au lieu d'utiliser les fichiers yaml")
+    parser.add_argument("--re2o-user", type=str, required=False, help="Permet de choisir l'user pour se connecter à l'api re2o, par défaut on prend l'user unix courant")
+    parser.add_argument("-4", "--force-ipv4", action="store_true", help="Force le provisionning en ipv4")
+
     args = parser.parse_args()
 
+    if args.re2o:
+       logger.debug("Loading master config")
+       master_config = yaml.load(open("configs/master.yml", "r"), yaml.Loader)
+       logger.debug("Loading config from re2o")
+       re2o_config = get_switch_from_re2o(args.re2o, args.switch_name, args.re2o_user)
+    else:
        logger.debug("Loading master config")
        master_config = yaml.load(open("configs/master.yml", "r"), yaml.Loader)
        logger.debug("Loading config for {}".format(args.switch_name))
@@ -195,17 +358,33 @@ if __name__ == "__main__":
 
     if args.host:
         switch_address = args.host
+    elif args.re2o:
+        if args.force_ipv4:
+            switch_address = re2o_config.get("ipv4")
+        else:
+            switch_address = re2o_config.get("ipv6") or re2o_config.get("ipv4")
+    else:
+        if args.force_ipv4:
+            switch_address = switch_config.get("ipv4-addr")
         else:
             switch_address = switch_config.get("ipv6-addr") or switch_config.get("ipv4-addr")
+
     if switch_address is None:
-        switch_address = "{}.switches.crans.org".format(args.switch_name)
+        switch_address = "{}.switches.crans.org".format(args.switch_name) #TODO: crans
 
     logger.info("Connecting to {} with address {}".format(args.switch_name, switch_address))
-    session = connect_to_switch(switch_address, user="root", key=master_config.get("ssh_private_key"))
+    session = connect_to_switch(switch_address, user="root", key=master_config.get("ssh_private_key")) #TODO: spécifier chemin clef
     old_config = sftp_read_file(session, "cfg/running-config").decode("utf-8")
+    header = get_header(old_config)
 
+    # génération de la conf
     logging.info("Generating configuration for {}".format(args.switch_name))
-    configuration = gen_conf(master_config, switch_config, old_config)
+    if args.re2o:
+        configuration = gen_conf_re2o(re2o_config, header)
+    else:
+        configuration = gen_conf(master_config, switch_config, header)
+
+    # génération du diff
     for line in difflib.unified_diff(old_config.split("\n"), configuration.split("\n"), fromfile='origin', tofile='new', lineterm=""):
         if line.startswith("-"):
             termcolor.cprint(line, "red")
@@ -225,3 +404,10 @@ if __name__ == "__main__":
     session = connect_to_switch(switch_address, user="crans", key=master_config.get("ssh_private_key"))
     sftp_write_file(session, "/tmp/conf_tmp", "cfg/startup-config")
     logging.info("Deployement done for {}".format(args.switch_name))
+
+
+# on retire ces deux lignes qui sont hardcodées dans les templates crans
+# c'est deux adresses de multicast au pif, wtf ?
+# https://gitlab.crans.org/nounous/scripts/-/commit/af2d158ff1a99dee2f032a81bf14de4144d2b82f
+#-filter multicast 01005e-0000fb drop 1-52
+#-filter multicast 333300-0000fb drop 1-52

requirements.txt

@@ -1,7 +1,12 @@
+certifi==2020.6.20
+chardet==3.0.4
 colorlog==4.2.1
+idna==2.10
 Jinja2==2.11.2
 MarkupSafe==1.1.1
 pkg-resources==0.0.0
 PyYAML==5.3.1
+requests==2.24.0
 ssh2-python==0.18.0.post1
 termcolor==1.1.0
+urllib3==1.25.10