[Flang] Ported test_symbols to Python

Due to unavailability of Flang testing on Windows, the shell scripts
are being ported to Python. The following changes are being made in
this patch: removed test_symbols.sh and common.sh, and ported them
to Python. Changes to the tests themselves so that they use the
python scripts instead of the shell script.

Reviewed By: Meinersbur, awarzynski, kiranchandramohan

Differential Revision: https://reviews.llvm.org/D107041

flang/test/Semantics/OpenACC/acc-symbols01.f90

@@ -1,5 +1,4 @@
-! RUN: %S/../test_symbols.sh %s %t %flang_fc1 -fopenacc
-! REQUIRES: shell
+! RUN: %python %S/../test_symbols.py %s %flang_fc1 -fopenacc
 
 !DEF: /mm MainProgram
 program mm

flang/test/Semantics/common.py

@@ -0,0 +1,44 @@
+"""Provides common functionality to the test scripts."""
+
+import os
+import sys
+from pathlib import Path
+
+def set_source(source):
+    """Checks whether the source file exists and returns its path."""
+    if not Path(source).is_file():
+        die(source)
+    return Path(source)
+
+def set_executable(executable):
+    """Checks whether a Flang executable has been set and returns its path."""
+    flang_fc1 = Path(executable)
+    if not flang_fc1.is_file():
+        die(flang_fc1)
+    return str(flang_fc1)
+
+def set_temp(tmp):
+    """Sets a temporary directory or creates one if it doesn't exist."""
+    os.makedirs(Path(tmp), exist_ok=True)
+    return Path(tmp)
+
+def die(file=None):
+    """Used in other functions."""
+    if file is None:
+        print(f"{sys.argv[0]}: FAIL")
+    else:
+        print(f"{sys.argv[0]}: File not found: {file}")
+    sys.exit(1)
+
+def check_args(args):
+    """Verifies that 2 arguments have been passed."""
+    if len(args) < 3:
+        print(f"Usage: {args[0]} <fortran-source> <flang-command>")
+        sys.exit(1)
+
+def check_args_long(args):
+    """Verifies that 3 arguments have been passed."""
+    if len(args) < 4:
+        print(f"Usage: {args[0]} <fortran-source> <temp-test-dir> <flang-command>")
+        sys.exit(1)
+

flang/test/Semantics/kinds01.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1
  !DEF: /MainProgram1/jk1 ObjectEntity INTEGER(1)
  integer(kind=1) jk1
  !DEF: /MainProgram1/js1 ObjectEntity INTEGER(1)

flang/test/Semantics/kinds03.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1
  !DEF: /MainProgram1/ipdt DerivedType
  !DEF: /MainProgram1/ipdt/k TypeParam INTEGER(4)
  type :: ipdt(k)

flang/test/Semantics/omp-do-schedule03.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1 -fopenmp
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1 -fopenmp
 ! OpenMP Version 4.5
 ! 2.7.1 Schedule Clause
 ! Test that does not catch non constant integer expressions like xx - xx.

flang/test/Semantics/omp-do-schedule04.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1 -fopenmp
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1 -fopenmp
 ! OpenMP Version 4.5
 ! 2.7.1 Schedule Clause
 ! Test that does not catch non constant integer expressions like xx - yy.

flang/test/Semantics/omp-do01-positivecase.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1 -fopenmp
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1 -fopenmp
 ! OpenMP Version 4.5
 ! 2.7.1 Loop Construct
 ! The loop iteration variable may not appear in a firstprivate directive.

flang/test/Semantics/omp-do04-positivecase.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1 -fopenmp
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1 -fopenmp
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop Constructs
 

flang/test/Semantics/omp-do05-positivecase.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1 -fopenmp
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1 -fopenmp
 ! OpenMP Version 4.5
 ! 2.7.1 Loop Construct restrictions on single directive.
 ! A positive case

flang/test/Semantics/omp-do06-positivecases.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1 -fopenmp
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1 -fopenmp
 ! OpenMP Version 4.5
 ! 2.7.1 Loop Construct
 ! The ordered clause must be present on the loop construct if any ordered

flang/test/Semantics/omp-do11.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1 -fopenmp
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1 -fopenmp
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop Constructs
 

flang/test/Semantics/omp-do12.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1 -fopenmp
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1 -fopenmp
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop constructs.
 

flang/test/Semantics/omp-do14.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1 -fopenmp
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1 -fopenmp
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop constructs.
 

flang/test/Semantics/omp-do17.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1 -fopenmp
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1 -fopenmp
 ! OpenMP Version 4.5
 ! 2.7.1 Do Loop constructs.
 

flang/test/Semantics/omp-reduction08.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1 -fopenmp
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1 -fopenmp
 ! OpenMP Version 4.5
 ! 2.15.3.6 Reduction Clause Positive cases
 

flang/test/Semantics/omp-reduction09.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1 -fopenmp
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1 -fopenmp
 ! OpenMP Version 4.5
 ! 2.15.3.6 Reduction Clause Positive cases.
 !DEF: /omp_reduction MainProgram

flang/test/Semantics/omp-symbol01.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1 -fopenmp
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1 -fopenmp
 
 ! Test clauses that accept list.
 ! 2.1 Directive Format

flang/test/Semantics/omp-symbol02.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1 -fopenmp
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1 -fopenmp
 
 ! 1.4.1 Structure of the OpenMP Memory Model
 

flang/test/Semantics/omp-symbol03.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1 -fopenmp
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1 -fopenmp
 
 ! 1.4.1 Structure of the OpenMP Memory Model
 ! In the inner OpenMP region, SHARED `a` refers to the `a` in the outer OpenMP

flang/test/Semantics/omp-symbol04.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1 -fopenmp
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1 -fopenmp
 
 ! 2.15.3 Data-Sharing Attribute Clauses
 ! Both PARALLEL and DO (worksharing) directives need to create new scope,

flang/test/Semantics/omp-symbol05.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1 -fopenmp
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1 -fopenmp
 
 ! 2.15.2 threadprivate Directive
 ! The threadprivate directive specifies that variables are replicated,

flang/test/Semantics/omp-symbol06.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1 -fopenmp
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1 -fopenmp
 
 ! 2.15.3 Data-Sharing Attribute Clauses
 ! A list item that specifies a given variable may not appear in more than

flang/test/Semantics/omp-symbol07.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1 -fopenmp
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1 -fopenmp
 
 ! Generic tests
 !   1. subroutine or function calls should not be fixed for DSA or DMA

flang/test/Semantics/omp-symbol08.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1 -fopenmp
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1 -fopenmp
 
 ! 2.15.1.1 Predetermined rules for associated do-loops index variable
 !   a) The loop iteration variable(s) in the associated do-loop(s) of a do,

flang/test/Semantics/procinterface01.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! Tests for "proc-interface" semantics.
 ! These cases are all valid.
 

flang/test/Semantics/symbol01.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! Test that intent-stmt and subprogram prefix and suffix are resolved.
 
 !DEF: /m Module

flang/test/Semantics/symbol02.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! Test host association in module subroutine and internal subroutine.
 
 !DEF: /m Module

flang/test/Semantics/symbol03.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! Test host association in internal subroutine of main program.
 
 !DEF: /main MainProgram

flang/test/Semantics/symbol05.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! Explicit and implicit entities in blocks
 
 !DEF: /s1 (Subroutine) Subprogram

flang/test/Semantics/symbol06.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1
 !DEF: /main MainProgram
 program main
  !DEF: /main/t1 DerivedType

flang/test/Semantics/symbol07.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1
 !DEF: /main MainProgram
 program main
  implicit complex(z)

flang/test/Semantics/symbol08.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1
 !DEF: /main MainProgram
 program main
  !DEF: /main/x POINTER ObjectEntity REAL(4)

flang/test/Semantics/symbol09.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1
 !DEF: /s1 (Subroutine) Subprogram
 subroutine s1
  !DEF: /s1/a ObjectEntity REAL(4)

flang/test/Semantics/symbol10.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1
 !DEF: /m1 Module
 module m1
 contains

flang/test/Semantics/symbol11.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1
 !DEF: /s1 (Subroutine) Subprogram
 subroutine s1
  implicit none

flang/test/Semantics/symbol12.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! Verify that SAVE attribute is propagated by EQUIVALENCE
 
 !DEF: /s1 (Subroutine) Subprogram

flang/test/Semantics/symbol13.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! Old-style "*length" specifiers (R723)
 
 !DEF: /f1 (Function) Subprogram CHARACTER(1_8,1)

flang/test/Semantics/symbol14.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! "Bare" uses of type parameters and components
 
  !DEF: /MainProgram1/t1 DerivedType

flang/test/Semantics/symbol15.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! Forward references in pointer initializers and TBP bindings.
 
 !DEF: /m Module

flang/test/Semantics/symbol16.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! Statement functions
 
 !DEF: /p1 MainProgram

flang/test/Semantics/symbol17.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1
 ! Forward references to derived types (non-error cases)
 
 !DEF: /main MainProgram

flang/test/Semantics/symbol18.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1
 
 ! Intrinsic function in type declaration statement: type is ignored
 

flang/test/Semantics/symbol19.f90

@@ -1,5 +1,4 @@
-! RUN: %S/test_symbols.sh %s %t %flang_fc1
-! REQUIRES: shell
+! RUN: %python %S/test_symbols.py %s %flang_fc1
 
 ! Test that a procedure is only implicitly resolved as an intrinsic function
 ! (resp. subroutine) if this is a function (resp. subroutine)

flang/test/Semantics/test_symbols.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+"""Compiles a source file with "-fdebug-unparse-with-symbols' and verifies
+we get the right symbols in the output, i.e. the output should be
+the same as the input, except for the copyright comment.
+Expects a source file passed as the first argument;
+Expects the Flang frontdriver with options as second argument."""
+
+import sys
+import re
+import subprocess
+import common as cm
+
+from difflib import unified_diff
+
+cm.check_args(sys.argv)
+src = cm.set_source(sys.argv[1])
+diff1 = ""
+diff2 = ""
+
+flang_fc1 = cm.set_executable(sys.argv[2])
+flang_fc1_args = sys.argv[3:]
+flang_fc1_options = "-fdebug-unparse-with-symbols"
+
+# Strips out blank lines and all comments except for "!DEF:", "!REF:", "!$acc" and "!$omp"
+with open(src, 'r') as text_in:
+    for line in text_in:
+        text = re.sub(r"!(?![DR]EF:|\$omp|\$acc).*", "", line)
+        text = re.sub(r"^\s*$", "", text)
+        diff1 += text
+
+# Strips out "!DEF:" and "!REF:" comments
+for line in diff1:
+    text = re.sub(r"![DR]EF:.*", "", line)
+    diff2 += text
+
+# Compiles, inserting comments for symbols:
+cmd = [flang_fc1, *flang_fc1_args, flang_fc1_options]
+diff3 = subprocess.check_output(cmd, input=diff2, universal_newlines=True)
+
+# Removes all whitespace to compare differences in files
+diff1 = diff1.replace(" ", "")
+diff3 = diff3.replace(" ", "")
+diff_check = ""
+
+# Compares the input with the output
+for line in unified_diff(diff1, diff3, n=999999,
+                         fromfile="Expected output", tofile="Actual output"):
+    diff_check += line
+
+if diff_check != "":
+    print(diff_check.replace(" ", ""))
+    print()
+    print("FAIL")
+    sys.exit(1)
+else:
+    print()
+    print("PASS")
+

flang/test/Semantics/test_symbols.sh

@@ -1,34 +0,0 @@
-#!/usr/bin/env bash
-# Compile a source file with '-fdebug-unparse-with-symbols' and verify
-# we get the right symbols in the output, i.e. the output should be
-# the same as the input, except for the copyright comment.
-# Change the frontend driver by setting the FLANG_FC1 environment variable.
-
-FLANG_FC1_OPTIONS="-fdebug-unparse-with-symbols"
-srcdir=$(dirname $0)
-source $srcdir/common.sh
-[[ ! -f $src ]] && echo "File not found: $src" && exit 1
-
-src1=$temp/1.f90
-src2=$temp/2.f90
-src3=$temp/3.f90
-diffs=$temp/diffs
-
-# Strip out blank lines and all comments except "!DEF:", "!REF:", and "!$omp"
-sed -e 's/!\([DR]EF:\)/KEEP \1/' -e 's/!\($omp\)/KEEP \1/' \
-  -e 's/!\($acc\)/KEEP \1/' -e 's/!.*//' -e 's/ *$//' -e '/^$/d' \
-  -e 's/KEEP \([DR]EF:\)/!\1/' -e 's/KEEP \($omp\)/!\1/' \
-  -e 's/KEEP \($acc\)/!\1/' \
-  $src > $src1
-egrep -v '![DR]EF:' $src1 > $src2  # strip out DEF and REF comments
-# compile, inserting comments for symbols:
-( cd $temp; $FLANG_FC1 $FLANG_FC1_OPTIONS $(basename $src2) ) > $src3
-
-if diff -w -U999999 $src1 $src3 > $diffs; then
-  echo PASS
-else
-  sed '1,/^\@\@/d' $diffs
-  echo
-  echo FAIL
-  exit 1
-fi