From 48b26acf0d2c902b424416b0c300b3198330e2d1 Mon Sep 17 00:00:00 2001 From: savagebidoof Date: Sat, 22 Apr 2023 05:00:19 +0200 Subject: [PATCH] Did the first authentication example --- .../06-mTLS/01-namespace.yaml | 7 - Istio/02-traffic_management/06-mTLS/README.md | 75 ++--- .../06-mTLS/deployment.yaml | 11 +- .../06-mTLS/deployment_2.yaml | 11 +- .../06-mTLS/gateway.yaml | 2 - .../01-namespaces/01-namespace.yaml | 7 + Istio/authentication/01-namespaces/README.md | 288 ++++++++++++++++++ .../01-namespaces/authentication.yaml | 54 ++++ .../01-namespaces/deployment.yaml | 40 +++ .../01-namespaces/deployment_2.yaml | 43 +++ .../authentication/01-namespaces/gateway.yaml | 45 +++ Istio/authentication/README.md | 26 ++ Istio/envoy/01-envoy_add_headers/README.md | 2 +- Istio/troubleshooting/README.md | 3 + 14 files changed, 533 insertions(+), 81 deletions(-) delete mode 100755 Istio/02-traffic_management/06-mTLS/01-namespace.yaml create mode 100755 Istio/authentication/01-namespaces/01-namespace.yaml create mode 100755 Istio/authentication/01-namespaces/README.md create mode 100644 Istio/authentication/01-namespaces/authentication.yaml create mode 100755 Istio/authentication/01-namespaces/deployment.yaml create mode 100755 Istio/authentication/01-namespaces/deployment_2.yaml create mode 100755 Istio/authentication/01-namespaces/gateway.yaml create mode 100644 Istio/authentication/README.md diff --git a/Istio/02-traffic_management/06-mTLS/01-namespace.yaml b/Istio/02-traffic_management/06-mTLS/01-namespace.yaml deleted file mode 100755 index 71be03c..0000000 --- a/Istio/02-traffic_management/06-mTLS/01-namespace.yaml +++ /dev/null @@ -1,7 +0,0 @@ -#apiVersion: v1 -#kind: Namespace -#metadata: -# name: foo -# labels: -# istio-injection: "enabled" -#--- \ No newline at end of file diff --git a/Istio/02-traffic_management/06-mTLS/README.md b/Istio/02-traffic_management/06-mTLS/README.md index 9f40550..921e3bd 100755 --- a/Istio/02-traffic_management/06-mTLS/README.md +++ b/Istio/02-traffic_management/06-mTLS/README.md @@ -1,66 +1,23 @@ -https://istio.io/latest/docs/concepts/security/#authentication-policies - -https://istio.io/latest/docs/tasks/security/authentication/mtls-migration/ - -https://istio.io/latest/docs/concepts/security/#mutual-tls-authentication - - # Continues from -- 01-hello_world_1_service_1_deployment - - - +- [01-hello_world_1_service_1_deployment](../../01-simple/01-hello_world_1_service_1_deployment) +## Description Nowadays, by default, Istio will have mTLS automatically enabled, allowing the Istio Sidecars to **automatically** negotiate the TLS traffic between them.encrypted -To avoid this behavior, the pod requires to not have a Istio Sidecar set to that pod, for that reason on this example we set up 2 deployments, 1 with a sidecar, and a second without a sidecar. +To avoid this behavior, the pod requires to not have an Istio Sidecar set to that pod, for that reason on this example we set up 2 deployments, 1 with a sidecar, and a second without a sidecar. From the Kiali dashboard we will review the mTLS label displayed > **Note:**\ > If the PeerAuthentication is deployed in the `istio-system` namespace, it will affect all the namespaces in the cluster. +# Changelog # Walkthrough - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ## Deploy the resources ```shell @@ -103,7 +60,7 @@ istioctl dashboard kiali ## Display services menu -![Kiali menu, displaying 3 services. helloworld, byeworld and kubernetes][./src/06-kiali-services.png] +![Kiali menu, displaying 3 services. helloworld, byeworld and kubernetes](../src/06-kiali-services.png) > **Highlight:**\ > On the column located at the right, we can notice a note saying `Missing Sidecar` @@ -116,17 +73,25 @@ istioctl dashboard kiali On the service `byeworld` (reminder that it's pods had the Istio sidecar injection disabled), it displays the message `No mTLS`, meaning that mTLS (Mutual TLS between Istio sidecards) is not available. -![][./src/06-kiali-services-byeworld.png] +![](../src/06-kiali-services-byeworld.png) ### Helloworld On the service `helloworld`, it displays the message `mTLS` -![][./src/06-kiali-services-helloworld.pngk] +![](../src/06-kiali-services-helloworld.png) ## Test resources ### Curl / LB requests / requests from external traffic +#### Get LB IP + +```shell +$ kubectl get svc istio-ingressgateway -n istio-system +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +istio-ingressgateway LoadBalancer 10.97.47.216 192.168.1.50 15021:31316/TCP,80:32012/TCP,443:32486/TCP 39h +``` + #### helloworld The service works as intended as we can reach the `helloworld` service. @@ -185,4 +150,12 @@ As the rule is no longer being set, and for such not being applied, the traffic ```shell $ kubectl exec -i -t "$(kubectl get pod -l app=byeworld | tail -n 1 | awk '{print $1}')" -- curl http://helloworld.default.svc.cluster.local:8080 | grep ".*" Welcome to nginx! -``` \ No newline at end of file +``` + +# Links of interest + +- https://istio.io/latest/docs/concepts/security/#authentication-policies + +- https://istio.io/latest/docs/concepts/security/#mutual-tls-authentication + +- https://istio.io/latest/docs/tasks/security/authentication/mtls-migration/ diff --git a/Istio/02-traffic_management/06-mTLS/deployment.yaml b/Istio/02-traffic_management/06-mTLS/deployment.yaml index 6039be0..0fb81b3 100755 --- a/Istio/02-traffic_management/06-mTLS/deployment.yaml +++ b/Istio/02-traffic_management/06-mTLS/deployment.yaml @@ -1,4 +1,3 @@ -# https://github.com/istio/istio/blob/master/samples/helloworld/helloworld.yaml apiVersion: v1 kind: Service metadata: @@ -14,13 +13,6 @@ spec: selector: app: helloworld --- -#apiVersion: v1 -#kind: ServiceAccount -#metadata: -# name: istio-helloworld -# labels: -# account: ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -37,13 +29,12 @@ spec: labels: app: helloworld spec: -# serviceAccountName: istio-helloworld containers: - name: helloworld image: nginx resources: requests: cpu: "100m" - imagePullPolicy: IfNotPresent #Always + imagePullPolicy: IfNotPresent ports: - containerPort: 80 diff --git a/Istio/02-traffic_management/06-mTLS/deployment_2.yaml b/Istio/02-traffic_management/06-mTLS/deployment_2.yaml index ded5740..ad21186 100755 --- a/Istio/02-traffic_management/06-mTLS/deployment_2.yaml +++ b/Istio/02-traffic_management/06-mTLS/deployment_2.yaml @@ -1,4 +1,3 @@ -# https://github.com/istio/istio/blob/master/samples/helloworld/helloworld.yaml apiVersion: v1 kind: Service metadata: @@ -13,13 +12,6 @@ spec: targetPort: 80 selector: app: byeworld -#--- -#apiVersion: v1 -#kind: ServiceAccount -#metadata: -# name: istio-helloworld -# labels: -# account: --- apiVersion: apps/v1 kind: Deployment @@ -38,13 +30,12 @@ spec: app: byeworld sidecar.istio.io/inject: "false" spec: -# serviceAccountName: istio-byeworld containers: - name: byeworld image: nginx resources: requests: cpu: "100m" - imagePullPolicy: IfNotPresent #Always + imagePullPolicy: IfNotPresent ports: - containerPort: 80 diff --git a/Istio/02-traffic_management/06-mTLS/gateway.yaml b/Istio/02-traffic_management/06-mTLS/gateway.yaml index cf24f1f..05f8ca6 100755 --- a/Istio/02-traffic_management/06-mTLS/gateway.yaml +++ b/Istio/02-traffic_management/06-mTLS/gateway.yaml @@ -1,4 +1,3 @@ -# https://github.com/istio/istio/blob/master/samples/helloworld/helloworld-gateway.yaml apiVersion: networking.istio.io/v1alpha3 kind: Gateway metadata: @@ -42,6 +41,5 @@ spec: host: byeworld.default.svc.cluster.local port: number: 9090 -# protocol: HTTPS rewrite: uri: "/" \ No newline at end of file diff --git a/Istio/authentication/01-namespaces/01-namespace.yaml b/Istio/authentication/01-namespaces/01-namespace.yaml new file mode 100755 index 0000000..e7a45bc --- /dev/null +++ b/Istio/authentication/01-namespaces/01-namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: foo + labels: + istio-injection: "enabled" +--- \ No newline at end of file diff --git a/Istio/authentication/01-namespaces/README.md b/Istio/authentication/01-namespaces/README.md new file mode 100755 index 0000000..07c0a5b --- /dev/null +++ b/Istio/authentication/01-namespaces/README.md @@ -0,0 +1,288 @@ +# Continues from + +[//]: # (- [01-hello_world_1_service_1_deployment](../../01-simple/01-hello_world_1_service_1_deployment)) +- [06-mTLS](../../02-traffic_management/06-mTLS) + +## Description + +Bla bla bla + +Configuration targeting namespaces + +# Changelog + +## Authentication configuration deployed + +### default namespace + +#### Allow nothing + +If the action is not specified, it will deploy the rule as "ALLOW". + +Here we are deploying a rule that allows the traffic that it matches, yet as it has no conditions, it will never match. + +```yaml +# Deny all requests to namespace default +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: allow-nothing + namespace: default +``` + +Citing the [Authorization Policy documentation from Istio](https://istio.io/latest/docs/reference/config/security/authorization-policy), regarding the evaluation behavior of this rules: + + 1. If there are any CUSTOM policies that match the request, evaluate and deny the request if the evaluation result is deny. + 2. If there are any DENY policies that match the request, deny the request. + 3. If there are no ALLOW policies for the workload, allow the request. + 4. If any of the ALLOW policies match the request, allow the request. + 5. Deny the request. + +On this scenario, as we don't have any DENY or CUSTOM rule, we skip right into the 3rd scenario. + +This rule is being applied to the workload (due being a rule that affects the whole namespace), and for such the 3rd scenario is not being applied either. + +On the 4rth, scenario, as the rule deployed, even if it's on ALLOW mode, has no conditions, it won't allow the traffic either. + +And finally, as any of the above scenarios allowed the traffic of the request, it ends getting denied. + +For such, the creation of this "empty" rule, has set the authorization mode on the not explicitly allowed request to "DENY ALL". + +### foo namespace + +#### Allow nothing + +Same behavior as above, this time applied to the namespace `foo` + +```yaml +# Deny all requests to namespace foo +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: allow-nothing + namespace: foo +spec: + {} +``` + + +#### allow-from-istio-system + +As we have a service deployed, and the traffic will come through the Istio Load Balancer (at least on my environment). I have set a rule that will allow all the traffic coming from a resource located in the namespace `istio-system`. + +```yaml +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: allow-from-istio-system + namespace: foo +spec: + action: ALLOW + rules: + - from: + - source: + namespaces: ["istio-system"] +``` + +#### allow-get-from-default + +As an additional example, I have set a new rule, that will allow the traffic comming from the namespace `default`, as long the method used is `HEAD` and is not targeting the path `/secret`. + +```yaml +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: allow-get-from-default + namespace: foo +spec: + action: ALLOW + rules: + - from: + - source: + namespaces: ["default"] + to: + - operation: + methods: ["HEAD"] + notPaths: ["/secret*"] +``` + +Citing the [`rule.source.namespaces` field from the Authorization Policy documentation from Istio](https://istio.io/latest/docs/reference/config/security/authorization-policy/#Source): + +> This field requires mTLS enabled and is the same as the source.namespace attribute. + +# Walkthrough + +## Deploy the resources + +```shell +$ kubectl apply -f ./ +namespace/foo created +authorizationpolicy.security.istio.io/allow-nothing created +authorizationpolicy.security.istio.io/allow-nothing created +authorizationpolicy.security.istio.io/allow-from-istio-system created +authorizationpolicy.security.istio.io/allow-get-from-default created +service/helloworld created +deployment.apps/helloworld-nginx created +service/byeworld created +deployment.apps/byeworld-nginx created +gateway.networking.istio.io/helloworld-gateway created +virtualservice.networking.istio.io/helloworld-vs created +``` + +## Test resources + +### Curl / LB requests / requests from external traffic + +#### Get LB IP + +```shell +$ kubectl get svc istio-ingressgateway -n istio-system +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +istio-ingressgateway LoadBalancer 10.97.47.216 192.168.1.50 15021:31316/TCP,80:32012/TCP,443:32486/TCP 39h +``` + +#### helloworld + +Due to the rule `allow-nothing` created on the namespace `default`, we are not hitting any rule that explicitly allows us, and for such, the traffic is being denied. + +For such we receive the status code `403` (**Forbidden**) + +```shell +$ curl 192.168.1.50/helloworld -I +HTTP/1.1 403 Forbidden +content-length: 19 +content-type: text/plain +date: Sat, 22 Apr 2023 02:00:34 GMT +server: istio-envoy +x-envoy-upstream-service-time: 11 +``` + +#### byeworld + +As we created the rule `allow-from-istio-system` created in the namespace `foo`, which allows all the traffic coming from a resource located in the namespace `istio-system`, and the load balancer used is located in the namespace `istio-system`, the traffic is allowed. + +For such we receive the code `200`. + +```shell +$ curl 192.168.1.50/byeworld --head +HTTP/1.1 200 OK +server: istio-envoy +date: Sat, 22 Apr 2023 02:01:48 GMT +content-type: text/html +content-length: 615 +last-modified: Tue, 28 Mar 2023 15:01:54 GMT +etag: "64230162-267" +accept-ranges: bytes +x-envoy-upstream-service-time: 91 +``` + +### Connectivity between the deployments + +> **NOTE:**\ +> The command `curl`, when uses the flag `--head` or `-I`, the request sent will be a `HEAD` request. +> +> It's important to be aware of that due the rule configured, where one of the targets was the method used, specifically targeted the method `HEAD`. + +#### helloworld towards byeworld (HEAD REQUEST) + +It works. + +Due to the rule `allow-get-from-default` deployed on the namespace `foo`, which allowed the traffic coming from the namespace `default` as long it used the method `HEAD` and wasn't targeting the path `/secret`, the request is allowed. + + + +```shell +$ kubectl exec -i -t "$(kubectl get pod -l app=helloworld | tail -n 1 | awk '{print $1}')" -- curl http://byeworld.foo.svc.cluster.local:9090 --head +HTTP/1.1 200 OK +server: envoy +date: Sat, 22 Apr 2023 02:08:56 GMT +content-type: text/html +content-length: 615 +last-modified: Tue, 28 Mar 2023 15:01:54 GMT +etag: "64230162-267" +accept-ranges: bytes +x-envoy-upstream-service-time: 6 +``` + +#### helloworld towards byeworld (GET REQUEST) + +(we removed the `--head` flag) + +It fails. + +Due to the rule `allow-get-from-default` deployed on the namespace `foo`, which allowed the traffic coming from the namespace `default` as long it used the method `HEAD` and wasn't targeting the path `/secret`, the request is allowed. + + +```shell +$ kubectl exec -i -t "$(kubectl get pod -l app=helloworld | tail -n 1 | awk '{print $1}')" -- curl http://byeworld.foo.svc.cluster.local:9090 +RBAC: access denied% +``` + +#### byeworld towards helloworld + +It fails. + +As expected, like when accessing through the Load Balancer, we receive the status code `403` (**Forbidden**). + +The `HEAD` request is irrelevant on this scenario, yet using it as I like this output more. + +```shell +$ kubectl exec -i -n foo -t "$(kubectl get pod -n foo -l app=byeworld | tail -n 1 | awk '{print $1}')" -- curl http://helloworld.default.svc.cluster.local:8080 --head +HTTP/1.1 403 Forbidden +content-length: 19 +content-type: text/plain +date: Sat, 22 Apr 2023 02:06:21 GMT +server: envoy +x-envoy-upstream-service-time: 65 +``` + +#### helloworld towards byeworld/secret (HEAD REQUEST) + +```shell +$ kubectl exec -i -t "$(kubectl get pod -l app=helloworld | tail -n 1 | awk '{print $1}')" -- curl http://byeworld.foo.svc.cluster.local:9090/secret --head +HTTP/1.1 403 Forbidden +content-length: 19 +content-type: text/plain +date: Sat, 22 Apr 2023 02:40:30 GMT +server: envoy +x-envoy-upstream-service-time: 3 +``` + + +#### helloworld towards byeworld/not-found + +```shell +$ kubectl exec -i -t "$(kubectl get pod -l app=helloworld | tail -n 1 | awk '{print $1}')" -- curl http://byeworld.foo.svc.cluster.local:9090/secret --head +HTTP/1.1 403 Forbidden +content-length: 19 +content-type: text/plain +date: Sat, 22 Apr 2023 02:40:30 GMT +server: envoy +x-envoy-upstream-service-time: 3 +``` + + +--- +## Delete the PeerAuthentication configuration set + + +```shell +$ kubectl delete peerauthentications.security.istio.io default-mtls +``` + +### connectivity between byeworld towards helloworld + +As the rule is no longer being set, and for such not being applied, the traffic from `byeworld` is able to reach the service `helloworld` without having the need to using mTLS. + +```shell +$ kubectl exec -i -t "$(kubectl get pod -l app=byeworld | tail -n 1 | awk '{print $1}')" -- curl http://helloworld.default.svc.cluster.local:8080 | grep ".*" +Welcome to nginx! +``` + + + + + +# Links of interest + +- https://istio.io/latest/docs/reference/config/security/authorization-policy/ \ No newline at end of file diff --git a/Istio/authentication/01-namespaces/authentication.yaml b/Istio/authentication/01-namespaces/authentication.yaml new file mode 100644 index 0000000..5214206 --- /dev/null +++ b/Istio/authentication/01-namespaces/authentication.yaml @@ -0,0 +1,54 @@ +# Deny all requests to namespace foo +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: allow-nothing + namespace: foo +spec: + {} +--- +# Deny all requests to namespace default +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: allow-nothing + namespace: default +spec: + {} +--- +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: allow-from-istio-system + namespace: foo +spec: + action: ALLOW + rules: + - from: + - source: + namespaces: ["istio-system"] +--- +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: allow-head-from-default + namespace: foo +spec: + action: ALLOW + rules: + - from: + - source: + namespaces: ["default"] + to: + - operation: + methods: ["HEAD"] + notPaths: ["/secret*"] +--- +apiVersion: security.istio.io/v1beta1 +kind: PeerAuthentication +metadata: + name: default-mtls + namespace: default +spec: + mtls: + mode: STRICT diff --git a/Istio/authentication/01-namespaces/deployment.yaml b/Istio/authentication/01-namespaces/deployment.yaml new file mode 100755 index 0000000..0fb81b3 --- /dev/null +++ b/Istio/authentication/01-namespaces/deployment.yaml @@ -0,0 +1,40 @@ +apiVersion: v1 +kind: Service +metadata: + name: helloworld + labels: + app: helloworld + service: helloworld +spec: + ports: + - port: 8080 + name: http + targetPort: 80 + selector: + app: helloworld +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: helloworld-nginx + labels: + app: helloworld +spec: + replicas: 1 + selector: + matchLabels: + app: helloworld + template: + metadata: + labels: + app: helloworld + spec: + containers: + - name: helloworld + image: nginx + resources: + requests: + cpu: "100m" + imagePullPolicy: IfNotPresent + ports: + - containerPort: 80 diff --git a/Istio/authentication/01-namespaces/deployment_2.yaml b/Istio/authentication/01-namespaces/deployment_2.yaml new file mode 100755 index 0000000..297b9d3 --- /dev/null +++ b/Istio/authentication/01-namespaces/deployment_2.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: Service +metadata: + name: byeworld + labels: + app: byeworld + service: byeworld + namespace: foo +spec: + ports: + - port: 9090 + name: http + targetPort: 80 + selector: + app: byeworld +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: byeworld-nginx + labels: + app: byeworld + namespace: foo +spec: + replicas: 1 + selector: + matchLabels: + app: byeworld + template: + metadata: + labels: + app: byeworld +# sidecar.istio.io/inject: "false" + spec: + containers: + - name: byeworld + image: nginx + resources: + requests: + cpu: "100m" + imagePullPolicy: IfNotPresent + ports: + - containerPort: 80 diff --git a/Istio/authentication/01-namespaces/gateway.yaml b/Istio/authentication/01-namespaces/gateway.yaml new file mode 100755 index 0000000..a481920 --- /dev/null +++ b/Istio/authentication/01-namespaces/gateway.yaml @@ -0,0 +1,45 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: helloworld-gateway +spec: + selector: + istio: ingressgateway # use istio default controller + servers: + - port: + number: 80 + name: http + protocol: HTTP + hosts: + - "*" +--- +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: helloworld-vs +spec: + hosts: + - "*" + gateways: + - helloworld-gateway + http: + - match: + - uri: + exact: /helloworld + route: + - destination: + host: helloworld.default.svc.cluster.local + port: + number: 8080 + rewrite: + uri: "/" + - match: + - uri: + exact: /byeworld + route: + - destination: + host: byeworld.foo.svc.cluster.local + port: + number: 9090 + rewrite: + uri: "/" \ No newline at end of file diff --git a/Istio/authentication/README.md b/Istio/authentication/README.md new file mode 100644 index 0000000..ba6ca50 --- /dev/null +++ b/Istio/authentication/README.md @@ -0,0 +1,26 @@ +## Authentication + +- Between pods + +- Between namespaces + +- Based on method + +- Based on service account(s) + +- Custom action (it's in alpha feature, should not focus on it for now) + +- Audit / logs + + + +reference (from specific deployment) + +https://discuss.istio.io/t/istio-deployment-deny-all-default/10983/6 + +```yaml + rules: + - from: + - source: + principals: ["cluster.local/ns/default/sa/bookinfo-reviews"] +``` \ No newline at end of file diff --git a/Istio/envoy/01-envoy_add_headers/README.md b/Istio/envoy/01-envoy_add_headers/README.md index a8952b6..74fa6b5 100755 --- a/Istio/envoy/01-envoy_add_headers/README.md +++ b/Istio/envoy/01-envoy_add_headers/README.md @@ -12,7 +12,7 @@ https://discuss.istio.io/t/adding-custom-response-headers-using-istios-1-6-0-env https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/lua_filter -kubectl logs -f deployments/istiod -n istio-system +> kubectl logs -f deployments/istiod -n istio-system diff --git a/Istio/troubleshooting/README.md b/Istio/troubleshooting/README.md index bcac82e..a531ed7 100644 --- a/Istio/troubleshooting/README.md +++ b/Istio/troubleshooting/README.md @@ -9,3 +9,6 @@ $ kubectl exec -n default "$(kubectl get pod -n default -l app1 =helloworld -o tcpdump: verbose output suppressed, use -v[v]... for full protocol decode listening on eth0, link-type EN10MB (Ethernet), snapshot length 262144 bytes ``` + + +### Logs \ No newline at end of file