Add first draft version of upmake, tool for updating makefiles.

Start moving away from files.bkl as the primary source for the files -- and
away from bakefile itself as the make/project file generator -- by storing the
list of files in a new build/files file and provide a simple build/upmake
script for updating files.bkl and the manually maintained MSVC10+ projects
from this file contents.

git-svn-id: https://svn.wxwidgets.org/svn/wx/wxWidgets/trunk@76610 c3d73ce0-8a6f-49c7-b76d-6d57e0e08775

build/tools/upmake/lib/Text/Upmake.pm

@@ -0,0 +1,78 @@
+package Text::Upmake;
+
+use strict;
+use warnings;
+use autodie;
+
+use Exporter qw(import);
+
+our @EXPORT = qw(read_files_list upmake);
+
+=head1 NAME
+
+Text::Upmake - Update make files.
+
+=head1 SYNOPSIS
+
+=head1 AUTHOR
+
+Vadim Zeitlin
+
+=cut
+
+# Reads the file containing the file lists definitions and returns a hash ref
+# with variable names as keys and refs to arrays of the file names as values.
+#
+# Takes an (open) file handle as argument.
+sub read_files_list
+{
+    my ($fh) = @_;
+
+    my ($var, %vars);
+    while (<$fh>) {
+        chomp;
+        s/#.*$//;
+        s/^\s+//;
+        s/\s+$//;
+        next if !$_;
+
+        if (/^(\w+)\s*=$/) {
+            $var = $1;
+        } else {
+            die "Unexpected contents outside variable definition at line $.\n"
+                unless defined $var;
+            push @{$vars{$var}}, $_;
+        }
+    }
+
+    return \%vars;
+}
+
+# Update the file with the given name in place using the specified function and
+# passing it the rest of the arguments.
+#
+# This is meant to be used with update_xxx() below.
+sub upmake
+{
+    my ($fname, $updater, @args) = @_;
+
+    my $fname_new = "$fname.upmake.new"; # TODO make it more unique
+
+    open my $in, '<', $fname;
+    open my $out, '>', $fname_new;
+
+    my $changed = $updater->($in, $out, @args);
+
+    close $in;
+    close $out;
+
+    if ($changed) {
+        rename $fname_new, $fname;
+    } else {
+        unlink $fname_new;
+    }
+
+    $changed
+}
+
+1;

build/tools/upmake/lib/Text/Upmake/Bakefile0.pm

@@ -0,0 +1,97 @@
+package Text::Upmake::Bakefile0;
+
+use Exporter qw(import);
+our @EXPORT = qw(update_bakefile_0);
+
+=head1 NAME
+
+Text::Upmake::Bakefile0 - Update bakefile-0.x files list.
+
+=head1 SYNOPSIS
+
+This is used exclusively to update wxWidgets C<files.bkl> and is probably not
+useful outside of wxWidgets project.
+
+    use Text::Upmake::Bakefile0;
+    Text::Upmake::upmake('bakefiles/files.bkl', \&update_bakefile_0, $vars);
+
+=head1 SEE ALSO
+
+Text::Upmake
+
+=head1 AUTHOR
+
+Vadim Zeitlin
+
+=cut
+
+# Update file with variable definitions in bakefile-0 format with the data
+# from the hash ref containing all the file lists.
+#
+# Takes the (open) file handles of the files to read and to write and the file
+# lists hash ref as arguments.
+#
+# Returns 1 if any changes were made.
+#
+# The caller must take care of actually renaming the second file to the first
+# one.
+sub update_bakefile_0
+{
+    my ($in, $out, $vars) = @_;
+
+    # Variable whose contents is being currently replaced.
+    my $var;
+
+    # Hash with files defined for the specified variable as keys and 0 or 1
+    # depending on whether we have seen them in the input file as values.
+    my %files;
+
+    # Set to 1 if we made any changes.
+    my $changed = 0;
+    while (<$in>) {
+        chomp;
+
+        if (/<set var="(\w+)" hints="files">/ && exists $vars->{$1}) {
+            $var = $1;
+            %files = map { $_ => 0 } @{$vars->{$var}};
+        } elsif (defined $var) {
+            local $_ = $_;
+            s/<!-- .* -->//;
+            s/^\s+//;
+            s/\s+$//;
+            if (m{</set>}) {
+                # Check if we have any new files.
+                #
+                # TODO Insert them in alphabetical order.
+                while (my ($file, $seen) = each(%files)) {
+                    if (!$seen) {
+                        # This file was wasn't present in the input, add it.
+                        # TODO Use proper indentation.
+                        print $out "    $file\n";
+
+                        $changed = 1;
+                    }
+                }
+
+                undef $var;
+            } elsif ($_) {
+                if (not exists $files{$_}) {
+                    # This file was removed.
+                    $changed = 1;
+                    next;
+                }
+
+                if ($files{$_}) {
+                    warn qq{Duplicate file "$_" in the definition of the } .
+                         qq{variable "$var" at line $.\n}
+                } else {
+                    $files{$_} = 1;
+                }
+            }
+        }
+
+        print $out "$_\n";
+    }
+
+    $changed
+}

build/tools/upmake/lib/Text/Upmake/MSBuild.pm

@@ -0,0 +1,252 @@
+package Text::Upmake::MSBuild;
+
+use Exporter qw(import);
+our @EXPORT = qw(update_msbuild update_msbuild_filters);
+
+=head1 NAME
+
+Text::Upmake::MSBuild - Update list of sources and headers in MSBuild projects.
+
+=head1 SYNOPSIS
+
+Given an MSBuild project C<project.vcxproj> and its associated filters file
+C<projects.vcxproj.filters>, the functions in this module can be used to update
+the list of files in them to correspond to the given ones.
+
+    use Text::Upmake::Bakefile0;
+    Text::Upmake::upmake('projects.vcxproj', \&update_msbuild, \@sources, \@headers);
+    Text::Upmake::upmake('projects.vcxproj.filters', \&update_msbuild_filters, \@sources, \@headers);
+
+=head1 SEE ALSO
+
+Text::Upmake
+
+=head1 AUTHOR
+
+Vadim Zeitlin
+
+=cut
+
+# Update sources and headers in an MSBuild project.
+#
+# Parameters: input and output file handles and array references to the sources
+# and the headers to be used in this project.
+#
+# Returns 1 if any changes were made.
+sub update_msbuild
+{
+    my ($in, $out, $sources, $headers) = @_;
+
+    # Hashes mapping the sources/headers names to 1 if they have been seen in
+    # the project or 0 otherwise.
+    my %sources = map { $_ => 0 } @$sources;
+    my %headers = map { $_ => 0 } @$headers;
+
+    # Reference to the hash corresponding to the files currently being
+    # processed.
+    my $files;
+
+    # Set to 1 when we are inside any <ItemGroup> tag.
+    my $in_group = 0;
+
+    # Set to 1 when we are inside an item group containing sources or headers
+    # respectively.
+    my ($in_sources, $in_headers) = 0;
+
+    # Set to 1 if we made any changes.
+    my $changed = 0;
+    while (my $line_with_eol = <$in>) {
+        (my $line = $line_with_eol) =~ s/\r?\n?$//;
+
+        if ($line =~ /^\s*<ItemGroup>$/) {
+            $in_group = 1;
+        } elsif ($line =~ m{^\s*</ItemGroup>$}) {
+            if (defined $files) {
+                my $kind = $in_sources ? 'Compile' : 'Include';
+
+                # Check if we have any new files.
+                #
+                # TODO Insert them in alphabetical order.
+                while (my ($file, $seen) = each(%$files)) {
+                    if (!$seen) {
+                        # Convert path separator to the one used by MSBuild.
+                        $file =~ s@/@\\@g;
+
+                        print $out qq{    <Cl$kind Include="$file" />\n};
+
+                        $changed = 1;
+                    }
+                }
+
+                $in_sources = $in_headers = 0;
+            }
+
+            $in_group = 0;
+        } elsif ($in_group) {
+            if ($line =~ m{^\s*<Cl(?<kind>Compile|Include) Include="(?<file>[^"]+)"\s*(?<slash>/)?>$}) {
+                if ($+{kind} eq 'Compile') {
+                    warn "Mix of sources and headers at line $.\n" if $in_headers;
+                    $in_sources = 1;
+                    $files = \%sources;
+                } else {
+                    warn "Mix of headers and sources at line $.\n" if $in_sources;
+                    $in_headers = 1;
+                    $files = \%headers;
+                }
+
+                my $closed_tag = defined $+{slash};
+
+                # Normalize the path separator, we always use Unix ones but the
+                # project files use Windows one.
+                my $file = $+{file};
+                $file =~ s@\\@/@g;
+
+                if (not exists $files->{$file}) {
+                    # This file was removed.
+                    $changed = 1;
+
+                    if (!$closed_tag) {
+                        # We have just the opening <ClCompile> tag, ignore
+                        # everything until the next </ClCompile>
+                        while (<$in>) {
+                            last if m{^\s*</ClCompile>$};
+                        }
+                    }
+
+                    # In any case skip either this line containing the full
+                    # <ClCompile/> tag or the line with the closing tag.
+                    next;
+                } else {
+                    if ($files->{$file}) {
+                        warn qq{Duplicate file "$file" in the project at line $.\n};
+                    } else {
+                        $files->{$file} = 1;
+                    }
+                }
+            }
+        }
+
+        print $out $line_with_eol;
+    }
+
+    $changed
+}
+
+# Update sources and headers in an MSBuild filters file.
+#
+# Parameters: input and output file handles, array references to the sources
+# and the headers to be used in this project and a callback used to determine
+# the filter for the new files.
+#
+# Returns 1 if any changes were made.
+sub update_msbuild_filters
+{
+    my ($in, $out, $sources, $headers, $filter_cb) = @_;
+
+    # Hashes mapping the sources/headers names to the text representing them in
+    # the input file if they have been seen in it or nothing otherwise.
+    my %sources = map { $_ => undef } @$sources;
+    my %headers = map { $_ => undef } @$headers;
+
+    # Reference to the hash corresponding to the files currently being
+    # processed.
+    my $files;
+
+    # Set to 1 when we are inside any <ItemGroup> tag.
+    my $in_group = 0;
+
+    # Set to 1 when we are inside an item group containing sources or headers
+    # respectively.
+    my ($in_sources, $in_headers) = 0;
+
+    # Set to 1 if we made any changes.
+    my $changed = 0;
+    while (my $line_with_eol = <$in>) {
+        (my $line = $line_with_eol) =~ s/\r?\n?$//;
+
+        if ($line =~ /^\s*<ItemGroup>?$/) {
+            $in_group = 1;
+        } elsif ($line =~ m{^\s*</ItemGroup>?$}) {
+            if (defined $files) {
+                # Output the group contents now, all at once, inserting any new
+                # files: we must do it like this to ensure that they are
+                # inserted in alphabetical order.
+                my $kind = $in_sources ? 'Compile' : 'Include';
+
+                foreach my $file (sort keys %$files) {
+                    if (defined $files->{$file}) {
+                        print $out $files->{$file};
+                    } else {
+                        my $filter = defined $filter_cb ? $filter_cb->($file) : undef;
+
+                        # Convert path separator to the one used by MSBuild.
+                        $file =~ s@/@\\@g;
+
+                        my $indent = ' ' x 2;
+
+                        print $out qq{$indent$indent<Cl$kind Include="$file"};
+                        if (defined $filter) {
+                            print $out ">\n$indent$indent$indent<Filter>$filter</Filter>\n$indent$indent</Cl$kind>\n";
+                        } else {
+                            print $out " />\n";
+                        }
+
+                        $changed = 1;
+                    }
+                }
+
+                $in_sources = $in_headers = 0;
+                $files = undef;
+            }
+
+            $in_group = 0;
+        } elsif ($in_group &&
+                 $line =~ m{^\s*<Cl(?<kind>Compile|Include) Include="(?<file>[^"]+)"\s*(?<slash>/)?>?$}) {
+            my $kind = $+{kind};
+            if ($kind eq 'Compile') {
+                warn "Mix of sources and headers at line $.\n" if $in_headers;
+                $in_sources = 1;
+                $files = \%sources;
+            } else {
+                warn "Mix of headers and sources at line $.\n" if $in_sources;
+                $in_headers = 1;
+                $files = \%headers;
+            }
+
+            my $closed_tag = defined $+{slash};
+
+            # Normalize the path separator, we always use Unix ones but the
+            # project files use Windows one.
+            my $file = $+{file};
+            $file =~ s@\\@/@g;
+
+            my $text = $line_with_eol;
+            if (!$closed_tag) {
+                # We have just the opening <ClCompile> tag, get everything
+                # until the next </ClCompile>.
+                while (<$in>) {
+                    $text .= $_;
+                    last if m{^\s*</Cl$kind>\r?\n?$};
+                }
+            }
+
+            if (not exists $files->{$file}) {
+                # This file was removed.
+                $changed = 1;
+            } else {
+                if ($files->{$file}) {
+                    warn qq{Duplicate file "$file" in the project at line $.\n};
+                } else {
+                    $files->{$file} = $text;
+                }
+            }
+
+            # Don't output this line yet, wait until the end of the group.
+            next
+        }
+
+        print $out $line_with_eol;
+    }
+
+    $changed
+}

build/tools/upmake/t/01_read_files_list.t

@@ -0,0 +1,24 @@
+use strict;
+use warnings;
+use autodie;
+use Test::More;
+
+BEGIN { use_ok('Text::Upmake'); }
+
+my $vars = read_files_list(*DATA);
+is_deeply($vars->{VAR1}, [qw(file1 file2)], 'VAR1 has expected value');
+is_deeply($vars->{VAR2}, [qw(file3 file4)], 'VAR2 has expected value');
+
+done_testing()
+
+__DATA__
+# Some comments
+
+VAR1 =
+    file1
+    # comment between the files
+    file2
+VAR2 =
+    file3
+    file4 # comment
+    # another comment

build/tools/upmake/t/02_update_bakefile_0.t

@@ -0,0 +1,47 @@
+use strict;
+use warnings;
+use autodie;
+use Test::More;
+
+BEGIN { use_ok('Text::Upmake::Bakefile0'); }
+
+my $vars = {
+        VAR1 => [qw(file1 file2 fileNew)],
+        VAR2 => [qw(file3 file4 file5 fileNew2)],
+    };
+
+open my $out, '>', \my $outstr;
+update_bakefile_0(*DATA, $out, $vars);
+
+note("Result: $outstr");
+
+like($outstr, qr/file1/, 'existing file was preserved');
+like($outstr, qr/fileNew$/m, 'new file was added');
+unlike($outstr, qr/fileOld/, 'old file was removed');
+like($outstr, qr/fileNew2/, 'another new file was added');
+like($outstr, qr/file3\s+file4/s, 'files remain in correct order');
+
+done_testing()
+
+__DATA__
+<?xml version="1.0" ?>
+<makefile>
+
+<!--
+    Some comment
+ -->
+
+<set var="VAR1" hints="files">
+    file1
+    <!-- comment between the files -->
+    file2
+</set>
+
+<set var="VAR2" hints="files">
+    file3
+    file4 <!-- comment after the file -->
+    file5
+    fileOld
+</set>
+
+</makefile>

build/tools/upmake/t/03_update_msbuild.t

@@ -0,0 +1,50 @@
+use strict;
+use warnings;
+use autodie;
+use Test::More;
+
+use Text::Upmake;
+BEGIN { use_ok('Text::Upmake::MSBuild'); }
+
+my $sources = [qw(file1.cpp file2.cpp fileNew.cpp)];
+my $headers = [qw(file1.h file2.h fileNew.h)];
+
+open my $out, '>', \my $outstr;
+update_msbuild(*DATA, $out, $sources, $headers);
+
+note("Result: $outstr");
+
+like($outstr, qr/file1\.cpp/, 'existing source file was preserved');
+like($outstr, qr/fileNew\.cpp/m, 'new source file was added');
+unlike($outstr, qr/fileOld\.cpp/, 'old source file was removed');
+unlike($outstr, qr/file3\.h/, 'old header was removed');
+like($outstr, qr/fileNew\.h/, 'new header was added');
+
+done_testing()
+
+__DATA__
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="file1.cpp" />
+    <ClCompile Include="file2.cpp" />
+    <ClCompile Include="fileOld.cpp" />
+    <ClCompile Include="file3.cpp" >
+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='DLL Debug|Win32'">Create</PrecompiledHeader>
+    </ClCompile>
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="file1.h" />
+    <ClInclude Include="file2.h" />
+    <ClInclude Include="file3.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>

build/tools/upmake/t/04_update_msbuild_filters.t

@@ -0,0 +1,67 @@
+use strict;
+use warnings;
+use autodie;
+use Test::More;
+
+use Text::Upmake;
+BEGIN { use_ok('Text::Upmake::MSBuild'); }
+
+my $sources = [qw(file1.cpp file2.cpp file4.cpp fileNew.cpp)];
+my $headers = [qw(file1.h file2.h fileNew.h)];
+
+sub filter_cb
+{
+    my ($file) = @_;
+
+    return 'New Sources' if $file =~ /New\./;
+
+    undef
+}
+
+open my $out, '>', \my $outstr;
+update_msbuild_filters(*DATA, $out, $sources, $headers, \&filter_cb);
+
+note("Result: $outstr");
+
+like($outstr, qr/file1\.cpp/, 'existing source file was preserved');
+like($outstr, qr/fileNew\.cpp/m, 'new source file was added');
+unlike($outstr, qr/fileOld\.cpp/, 'old source file was removed');
+unlike($outstr, qr/file3\.cpp/, 'another old source file was removed');
+unlike($outstr, qr/file3\.h/, 'old header was removed');
+like($outstr, qr/fileNew\.h/, 'new header was added');
+
+done_testing()
+
+__DATA__
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup>
+    <Filter Include="Common Sources">
+      <UniqueIdentifier>{...}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="Common Headers">
+      <UniqueIdentifier>{...}</UniqueIdentifier>
+    </Filter>
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="fileOld.cpp">
+      <Filter>Common Sources</Filter>
+    </ClCompile>
+    <ClCompile Include="file1.cpp">
+      <Filter>Common Sources</Filter>
+    </ClCompile>
+    <ClCompile Include="file2.cpp" />
+    <ClCompile Include="file3.cpp" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="file1.h">
+      <Filter>Common Headers</Filter>
+    </ClInclude>
+    <ClInclude Include="file2.h">
+      <Filter>Common Headers</Filter>
+    </ClInclude>
+    <ClInclude Include="file3.h">
+      <Filter>Common Headers</Filter>
+    </ClInclude>
+  </ItemGroup>
+</Project>

docs/tech/tn0016.txt

@@ -77,6 +77,12 @@ Files used to build the library are:
 3. Adding files to existing library
 -----------------------------------
 
+UPDATE: files.bkl is now itself partially generated from the master file
+        build/files. If the variable which you need to modify, according to the
+        instructions below, is already defined in build/files, update it there
+        and run build/upmake to update files.bkl.
+
+
 All files used by main libraries are listed in files.bkl. The file is
 organized into variables for toolkits, platforms and libraries. The variables
 come in pairs: there's always FOO_SRC for source files and FOO_HDR for header