playbook.yml

---

- name: Build Windows server.
  hosts: all
  gather_facts: false
  tasks:
    - name: install all critical and security updates
      win_updates:
        category_names:
          - CriticalUpdates
          - SecurityUpdates
          - UpdateRollups
        state: installed

    - name: Create directory structure for custom install packages
      win_file:
        path: C:\tmp\install
        state: directory

    - name: Create data-transfer folder
      win_file:
        path: C:\data-transfer
        state: directory

    - name: Create scripts folder
      win_file:
        path: C:\scripts
        state: directory

    - name: Create a Data feed folder on C drive
      win_file:
        path: 'C:\DATA FEED COPY'
        state: directory

    - name:  Create job export folder on C drive
      win_file:
        path: C:\Job_Export
        state: directory

    - name:  Create XML processing temp folder
      win_file:
        path: C:\XML_staging
        state: directory

    - name: Create PSTools directory
      win_file:
        path: C:\PSTools
        state: directory

    - name: Download PsTools
      win_get_url:
        url: https://download.sysinternals.com/files/PSTools.zip
        dest: C:\tmp\install\PSTools.zip

    - name: Unzip PsTools
      win_unzip:
        src: C:\tmp\install\PSTools.zip
        dest: C:\PSTools\
        delete_archive: yes

    - name: Download AWSCLI
      win_get_url:
        url: https://s3.amazonaws.com/aws-cli/AWSCLI64.msi
        dest: C:\tmp\install\AWSCLI64.msi

    - name: AWS CLI install
      win_package:
        path: C:\tmp\install\AWSCLI64.msi
        state: present

    - name:  GET config files from S3
      win_shell: |
        aws s3 cp s3://dq-config-bucket/dq-data-ingest-server/ C:\tmp\install\ --recursive
      args:
        executable: powershell.exe

    - name: Convert line endings from Linux to Windows (PowerShell)
      win_shell: '(Get-Content "C:\tmp\install\partitioning.txt") -replace "`n", "`r`n" | Set-Content "C:\tmp\install\volume_partitioning.txt"'

    - name: Run diskpart
      win_shell: diskpart /s C:\tmp\install\volume_partitioning.txt
      args:
        executable: cmd.exe

    - name: Create D drive folder structure
      win_file:
        path: D:\Releases
        state: directory

    - name: Install Cholocatey
      win_shell: |
        iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
        choco feature enable -n allowGlobalConfirmation

    - name: Install python2.7
      win_chocolatey:
        name: python2
        state: present

    - name: Install python3
      win_chocolatey:
        name: python3
        state: present

    - name: Ensure that Python2.7 and Python3.6 are present on the global system path
      win_path:
        elements:
        - 'C:\Python36'
        - 'C:\Python27'
        - 'C:\data-transfer'
        - 'C:\PSTools'

    - name: Install Python36 modules
      win_shell: pip3.exe install pyodbc psycopg2
      args:
        executable: cmd.exe

    - name: Install putty and plink
      win_chocolatey:
        name: putty
        state: present

    - name: Setup E drive folder structure
      win_command: C:\tmp\install\folder_structure.bat

    - name: Install PIP for Python27
      win_shell: |
        C:\Python27\python.exe C:\tmp\install\get-pip.py

    - name: Install GP loader
      win_package:
        path: C:\tmp\install\greenplum-loaders-4.3.4.0-build-1-WinXP-x86_32.msi
        state: present

    - name: Download PGAdmin3 1.18.1
      win_get_url:
        url: https://ftp.postgresql.org/pub/pgadmin/pgadmin3/v1.18.1/win32/pgadmin3-1.18.1.zip
        dest: C:\tmp\install\pgadmin3-1.18.1.zip

    - name: Unzip PGAdmin 1.18.1
      win_unzip:
        src: C:\tmp\install\pgadmin3-1.18.1.zip
        dest: C:\tmp\install\pgadmin3-1.18.1
        delete_archive: yes

    - name: Install PGAdmin 1.18.1
      win_package:
        path: C:\tmp\install\pgadmin3-1.18.1\pgadmin3.msi
        state: present

    - name: Download PSQLODBC driver
      win_get_url:
        url: https://ftp.postgresql.org/pub/odbc/versions/msi/psqlodbc_10_01_0000-x64.zip
        dest: C:\tmp\install\psqlodbc_10_01_0000-x64.zip

    - name: Unzip PSQLODBC driver
      win_unzip:
        src: C:\tmp\install\psqlodbc_10_01_0000-x64.zip
        dest: C:\tmp\install\psqlodbc_10_01_0000-x64
        delete_archive: yes

    - name: Install PSQL ODBC driver
      win_package:
        path: C:\tmp\install\psqlodbc_10_01_0000-x64\psqlodbc_x64.msi
        state: present

    - name: Download Python 2.5.4
      win_get_url:
        url: https://www.python.org/ftp/python/2.5.4/python-2.5.4.msi
        dest: C:\tmp\install\python-2.5.4.msi

    - name: Install Python 2.5.4
      win_package:
        path: C:\tmp\install\python-2.5.4.msi
        state: present

    - name: Install data-transfer pre-reqs
      win_command: pip3.exe install virtualenv virtualenvwrapper-win sqlalchemy==1.0.14 coverage==4.4.2 pytest==3.2.5 pytest-cov==2.5.1 pathlib==1.0.1 boto3==1.4.8 schedule==0.5.0 paramiko==2.4.0
      args:
        executable: cmd.exe

    - name: Set PYTHONPATH
      win_path:
        name: PYTHONPATH
        elements:
        - 'C:\Python27'
        - 'C:\Python25'
        - 'C:\Python36'
        - '%GPHOME_LOADERS%bin\lib'
        scope: machine
        state: present

    - name: Install modules for Python27
      win_shell: |
        pip2 install pyodbc pygresql paramiko psycopg2
      args:
        executable: cmd.exe

    - name: Download data-transfer from Github
      win_get_url:
        url: https://github.com/UKHomeOffice/data-transfer/archive/master.zip
        dest: C:\tmp\install\data-transfer-master.zip

    - name: Unzip data-transfer
      win_unzip:
        src: C:\tmp\install\data-transfer-master.zip
        dest: C:\data-transfer\
        delete_archive: yes

    - name: Create virtualenv for data-transfer
      win_shell: mkvirtualenv -p C:\Python36\python.exe data-transfer

    - name: Convert line endings from Linux to Windows (PowerShell)
      win_shell: '(Get-Content "C:\tmp\install\data_transfer.bat") -replace "`n", "`r`n" | Set-Content "C:\scripts\data_transfer.bat"'

    - name: Setup scheduled task for Data-transfer
      win_shell: |
        $action = New-ScheduledTaskAction -Execute C:\scripts\data_transfer_config.bat
        $trigger = New-ScheduledTaskTrigger -Daily -At (Get-Date)
        $description = "Run Data-transfer every minute forever."
        $principal = New-ScheduledTaskPrincipal -UserID "NT AUTHORITY\SYSTEM" -LogonType ServiceAccount -RunLevel Highest
        $task = Register-ScheduledTask -TaskName "Data_Transfer_S3_to_local" -Trigger $trigger -Action $action -Description $description -Principal $principal
        $task.Triggers.Repetition.Duration = "P1D"
        $task.Triggers.Repetition.Interval = "PT1M"
        $task | Set-ScheduledTask
        Disable-ScheduledTask -TaskName Data_Transfer_S3_to_local

    - name: Download scripts repo from Github
      win_get_url:
        url: https://github.com/UKHomeOffice/dq-xml_processing/archive/master.zip
        dest: C:\tmp\install\dq-xml_processing-master.zip

    - name: Unzip XML processing
      win_unzip:
        src: C:\tmp\install\dq-xml_processing-master.zip
        dest: C:\XML_staging
        delete_archive: yes

    - name: Move XML processing files
      win_shell: |
        $extractPath = "C:\XML_staging"
        $destinationPath = "E:\dq\nrt\s4_file_ingest\scripts"
        Get-ChildItem $extractPath\dq-xml_processing-master | Copy -Destination $destinationPath -Recurse
        Remove-Item $extractPath -Recurse -Force

    - name: Setup scheduled task for SEQ-Check
      win_shell: |
        $action = New-ScheduledTaskAction -Execute C:\Python36\python.exe E:\dq\nrt\s4_file_ingest\scripts\DQ_IL2_Seq_Check.py
        $trigger = New-ScheduledTaskTrigger -Daily -At (Get-Date)
        $description = "Run Seq job every minute forever."
        $principal = New-ScheduledTaskPrincipal -UserID "NT AUTHORITY\SYSTEM" -LogonType ServiceAccount -RunLevel Highest
        $task = Register-ScheduledTask -TaskName "SEQ_check" -Trigger $trigger -Action $action -Description $description -Principal $principal
        $task.Triggers.Repetition.Duration = "P1D"
        $task.Triggers.Repetition.Interval = "PT1M"
        $task | Set-ScheduledTask
        Disable-ScheduledTask -TaskName SEQ_check

    - name: Setup scheduled task for GA file transfer
      win_shell: |
        $action = New-ScheduledTaskAction -Execute C:\scripts\data_transfer_ga_config.bat
        $trigger = New-ScheduledTaskTrigger -Daily -At (Get-Date)
        $description = "Run Data-transfer to ACP S3 export every minute forever."
        $principal = New-ScheduledTaskPrincipal -UserID "NT AUTHORITY\SYSTEM" -LogonType ServiceAccount -RunLevel Highest
        $task = Register-ScheduledTask -TaskName "Data_Transfer_GA_to_ACP_S3" -Trigger $trigger -Action $action -Description $description -Principal $principal
        $task.Triggers.Repetition.Duration = "P1D"
        $task.Triggers.Repetition.Interval = "PT1M"
        $task | Set-ScheduledTask
        Disable-ScheduledTask -TaskName Data_Transfer_GA_to_ACP_S3

    - name: Setup scheduled task for Parsed Archive transfer
      win_shell: |
        $action = New-ScheduledTaskAction -Execute C:\scripts\data_transfer_parsed_archive_config.bat
        $trigger = New-ScheduledTaskTrigger -Daily -At (Get-Date)
        $description = "Run Parsed Data-transfer to Archive bucket every minute forever."
        $principal = New-ScheduledTaskPrincipal -UserID "NT AUTHORITY\SYSTEM" -LogonType ServiceAccount -RunLevel Highest
        $task = Register-ScheduledTask -TaskName "Parsed_Data_Transfer_Archive_to_S3" -Trigger $trigger -Action $action -Description $description -Principal $principal
        $task.Triggers.Repetition.Duration = "P1D"
        $task.Triggers.Repetition.Interval = "PT1M"
        $task | Set-ScheduledTask
        Disable-ScheduledTask -TaskName Parsed_Data_Transfer_Archive_to_S3

    - name: Setup scheduled task for Failed Archive transfer
      win_shell: |
        $action = New-ScheduledTaskAction -Execute C:\scripts\data_transfer_failed_archive_config.bat
        $trigger = New-ScheduledTaskTrigger -Daily -At (Get-Date)
        $description = "Run Failed Data-transfer to Archive bucket every minute forever."
        $principal = New-ScheduledTaskPrincipal -UserID "NT AUTHORITY\SYSTEM" -LogonType ServiceAccount -RunLevel Highest
        $task = Register-ScheduledTask -TaskName "Failed_Data_Transfer_Archive_to_S3" -Trigger $trigger -Action $action -Description $description -Principal $principal
        $task.Triggers.Repetition.Duration = "P1D"
        $task.Triggers.Repetition.Interval = "PT1M"
        $task | Set-ScheduledTask
        Disable-ScheduledTask -TaskName Failed_Data_Transfer_Archive_to_S3

    - name: Setup scheduled task for Stored Archive transfer
      win_shell: |
        $action = New-ScheduledTaskAction -Execute C:\scripts\data_transfer_stored_archive_config.bat
        $trigger = New-ScheduledTaskTrigger -Daily -At (Get-Date)
        $description = "Run Stored Data-transfer to Archive bucket every minute forever."
        $principal = New-ScheduledTaskPrincipal -UserID "NT AUTHORITY\SYSTEM" -LogonType ServiceAccount -RunLevel Highest
        $task = Register-ScheduledTask -TaskName "Stored_Data_Transfer_Archive_to_S3" -Trigger $trigger -Action $action -Description $description -Principal $principal
        $task.Triggers.Repetition.Duration = "P1D"
        $task.Triggers.Repetition.Interval = "PT1M"
        $task | Set-ScheduledTask
        Disable-ScheduledTask -TaskName Stored_Data_Transfer_Archive_to_S3

    - name: Setup scheduled task for RAW Archive transfer
      win_shell: |
        $action = New-ScheduledTaskAction -Execute C:\scripts\data_transfer_raw_archive_config.bat
        $trigger = New-ScheduledTaskTrigger -Daily -At (Get-Date)
        $description = "Run Data-transfer of RAW to Archive bucket every minute forever."
        $principal = New-ScheduledTaskPrincipal -UserID "NT AUTHORITY\SYSTEM" -LogonType ServiceAccount -RunLevel Highest
        $task = Register-ScheduledTask -TaskName "Data_Transfer_RAW_Archive_to_S3" -Trigger $trigger -Action $action -Description $description -Principal $principal
        $task.Triggers.Repetition.Duration = "P1D"
        $task.Triggers.Repetition.Interval = "PT1M"
        $task | Set-ScheduledTask
        Disable-ScheduledTask -TaskName Data_Transfer_RAW_Archive_to_S3

    - name: Firewall rules
      win_shell: |
        Set-NetFirewallProfile -Profile Domain -Enabled False
        Set-NetFirewallRule -Group "@FirewallAPI.dll,-28502" -Direction Inbound -Profile Domain -Enabled True
        New-NetFirewallRule -DisplayName "Wherescape Port 22 inbound" -Direction Inbound -Protocol TCP -LocalPort 22 -RemoteAddress 10.1.8.0/24 -Action Allow
        New-NetFirewallRule -DisplayName "Gpfdist TCP" -Direction Inbound -Protocol TCP -LocalPort Any -RemoteAddress Any -Program "C:\Program Files (x86)\Greenplum\greenplum-loaders-4.3.4.0-build-1\bin\gpfdist.exe"-Profile Public -Action Block
        New-NetFirewallRule -DisplayName "Gpfdist UDP" -Direction Inbound -Protocol UDP -LocalPort Any -RemoteAddress Any -Program "C:\Program Files (x86)\Greenplum\greenplum-loaders-4.3.4.0-build-1\bin\gpfdist.exe"-Profile Public -Action Block

    - name: Rename computer
      win_shell: |
        Rename-Computer -NewName DATA-INGEST1

    - name: Unzip Cloudwatch config file
      win_unzip:
        src: C:\tmp\install\AWS.EC2.Windows.Cloudwatch.json.zip
        dest: C:\Program Files\Amazon\SSM\Plugins\awsCloudWatch
        delete_archive: yes

    - name: Save CW config by restarting the Service
      win_shell: |
        Restart-Service AmazonSSMAgent