Unexpected character ''

Question

Upon migrating from VS2013 to VS2017 i'm running into an unexpected character issue.

Everything builds fine in msbuild 12.0 and VS2013, but when moving to 15.0 I receive hundreds of:

CS1519 Invalid token '?' in class, struct, or interface member declaration

in msbuild command line.

Building inside VS2017 returns:

CS1056 Unexpected character ''

var businessRuleData = principal.GetBusinessRule(​
BusinessRuleEnum.CONTENT_REPOSITORY);

Error occurs at Ch66 which is (B in between that area. The character that is hidden becomes a ? in WordPad. However, as mentioned the same code builds fine in msbuild 12.0. Deletion of all code and re-downloading form TFS didn't solve the issue

Solution Code

Note: Search in code for change_me and make sure to change to whatever your desired items are.

using Microsoft.TeamFoundation.Client;
using Microsoft.TeamFoundation.VersionControl.Client;
using System;
using System.Collections.Generic;
using System.IO;
using System.Text.RegularExpressions;

namespace FixZeroWidthSpace
{
    class Program
    {
        static void Main(string[] args)
        {
            // change to your directory
            var files = Directory.GetFiles(@"D:\change_me", "*.cs", SearchOption.AllDirectories);
            var counter = 0;
            var counterEdited = 0;
            var totalFiles = files.Length;
            var failed = new List<string>();
            var found = new List<string>();
            TfsTeamProjectCollection tpc = null;
            Workspace ws = null;

            foreach (var file in files)
            {
                if(counter % 10 == 0)
                {
                    Console.WriteLine("Searched {0} or {1} files, {2} have been edited.", counter, totalFiles, counterEdited);
                }
                // change to any folders you want to ignore or remove if none
                if (!file.Contains("change_me_ignore_folder_name"))
                {
                    string text = File.ReadAllText(file);
                    var regex = new Regex("[\u200B-\u200D\uFEFF]");
                    var newText = regex.Replace(text, "");

                    if (text != newText)
                    {
                        try
                        {
                            if (ws == null || tpc == null)
                            {
                                // change to your TFS server
                                tpc = new TfsTeamProjectCollection(new Uri(@"http://change_me_your_tfs_url/tfs/DefaultCollection"));
                                ws = FindWorkspaceByPath(tpc, file);
                            }

                            FileAttributes attributes = File.GetAttributes(file);

                            if ((attributes & FileAttributes.Hidden) == FileAttributes.Hidden)
                            {
                                attributes = RemoveAttribute(attributes, FileAttributes.Hidden);
                                File.SetAttributes(file, attributes);
                            }

                            ws.PendEdit(file);

                            if ((attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly)
                            {
                                // Make the file RW
                                attributes = RemoveAttribute(attributes, FileAttributes.ReadOnly);
                                File.SetAttributes(file, attributes);
                            }

                            File.WriteAllText(file, newText);
                            found.Add(file);
                            counterEdited++;
                        }
                        catch(Exception ex)
                        {
                            failed.Add(file);
                        }
                    }
                }
                counter++;
            }

            tpc.Dispose();
            File.WriteAllLines(Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location) + "\\found.txt", found);
            File.WriteAllLines(Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location) + "\\failed.txt", failed);

            Console.WriteLine("Press any key to exit...");
            Console.ReadKey();
        }

        private static FileAttributes RemoveAttribute(FileAttributes attributes, FileAttributes attributesToRemove)
        {
            return attributes & ~attributesToRemove;
        }

        private static Workspace FindWorkspaceByPath(TfsTeamProjectCollection tfs, string workspacePath)
        {
            VersionControlServer versionControl = tfs.GetService<VersionControlServer>();
            WorkspaceInfo workspaceInfo = Workstation.Current.GetLocalWorkspaceInfo(workspacePath);

            if (workspaceInfo != null)
            {
                return versionControl.GetWorkspace(workspaceInfo);
            }

            // No Workspace found using method 1, try to query all workspaces the user has on this machine.
            Workspace[] workspaces = versionControl.QueryWorkspaces(null, Environment.UserName, Environment.MachineName);
            foreach (Workspace w in workspaces)
            {
                foreach (WorkingFolder f in w.Folders)
                {
                    if (f.LocalItem.Equals(workspacePath))
                    {
                        return w;
                    }
                }
            }
            if (workspaces.Length > 0)
            {
                return workspaces[0];
            }

            throw new Exception(String.Format("TFS Workspace cannot be determined for {0}.", workspacePath));
        }
    }
}

Answer 1

Solution

Remove all invalid characters as MSBuild 15.0 and VS2017 are more strict on these unicode characters.

The following code can be utilized to accomplish the removal of all code in a Folder that is an issue. I utilized this as the changes required were too large to be done by hand.

---

C# Code

Variables

[Insert Folder to Scan] : Example C:\TFS\Code\Branch\Folder
[Insert Folder To Ignore] : Example 3rdPartyCode

Code

using System;
using System.Collections.Generic;
using System.IO;
using System.Text.RegularExpressions;

namespace FixZeroWidthSpace
{
    class Program
    {
        static void Main(string[] args)
        {
            var files = Directory.GetFiles(@"D:\TFS\210-219\212\MCET_212", "*.cs", SearchOption.AllDirectories);
            var counter = 0;
            var counterEdited = 0;
            var totalFiles = files.Length;
            var failed = new List<string>();
            var found = new List<string>();

            foreach (var file in files)
            {
                if(counter % 10 == 0)
                {
                    Console.WriteLine("Searched {0} or {1} files, {2} have been edited.", counter, totalFiles, counterEdited);
                }
                if (!file.Contains("[Insert Folder To Ignore]"))
                {
                    string text = File.ReadAllText(file);
                    var regex = new Regex("[\u200B-\u200D\uFEFF]");
                    var newText = regex.Replace(text, "");

                    if (text != newText)
                    {
                        try
                        {
                            FileAttributes attributes = File.GetAttributes(file);

                            if ((attributes & FileAttributes.Hidden) == FileAttributes.Hidden)
                            {
                                attributes = RemoveAttribute(attributes, FileAttributes.Hidden);
                                File.SetAttributes(file, attributes);
                            }

                            if ((attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly)
                            {
                                // Make the file RW
                                attributes = RemoveAttribute(attributes, FileAttributes.ReadOnly);
                                File.SetAttributes(file, attributes);
                            }

                            File.WriteAllText(file, newText);
                            found.Add(file);
                            counterEdited++;
                        }
                        catch(Exception ex)
                        {
                            failed.Add(file);
                        }
                    }
                }
                counter++;
            }

            File.WriteAllLines(Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location) + "\\found.txt", found);
            File.WriteAllLines(Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location) + "\\failed.txt", failed);

            Console.WriteLine("Press any key to exit...");
            Console.ReadKey();
        }

        private static FileAttributes RemoveAttribute(FileAttributes attributes, FileAttributes attributesToRemove)
        {
            return attributes & ~attributesToRemove;
        }
    }
}

---

C# Code w/ TFS Checkout

Variables

[Insert Folder to Scan] : Example C:\TFS\Code\Branch\Folder
[Insert Folder To Ignore] : Example 3rdPartyCode
[Insert URI to TFS Server Collection] : Example http://tfs.company.com:8080/tfs/DefaultCollection

Code

using Microsoft.TeamFoundation.Client;
using Microsoft.TeamFoundation.VersionControl.Client;
using System;
using System.Collections.Generic;
using System.IO;
using System.Text.RegularExpressions;

namespace FixZeroWidthSpace
{
    class Program
    {
        static void Main(string[] args)
        {
            var files = Directory.GetFiles(@"[Insert Folder to Scan]", "*.cs", SearchOption.AllDirectories);
            var counter = 0;
            var counterEdited = 0;
            var totalFiles = files.Length;
            var failed = new List<string>();
            var found = new List<string>();
            TfsTeamProjectCollection tpc = null;
            Workspace ws = null;



            foreach (var file in files)
            {
                if(counter % 10 == 0)
                {
                    Console.WriteLine("Searched {0} or {1} files, {2} have been edited.", counter, totalFiles, counterEdited);
                }
                if (!file.Contains("3rdparty"))
                {
                    string text = File.ReadAllText(file);
                    var regex = new Regex("[\u200B-\u200D\uFEFF]");
                    var newText = regex.Replace(text, "");

                    if (text != newText)
                    {
                        try
                        {
                            if (ws == null || tpc == null)
                            {
                                tpc = new TfsTeamProjectCollection(new Uri(@"[Insert URI to TFS Server Collection]"));
                                ws = FindWorkspaceByPath(tpc, file);
                            }

                            FileAttributes attributes = File.GetAttributes(file);

                            if ((attributes & FileAttributes.Hidden) == FileAttributes.Hidden)
                            {
                                attributes = RemoveAttribute(attributes, FileAttributes.Hidden);
                                File.SetAttributes(file, attributes);
                            }

                            ws.PendEdit(file);

                            if ((attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly)
                            {
                                // Make the file RW
                                attributes = RemoveAttribute(attributes, FileAttributes.ReadOnly);
                                File.SetAttributes(file, attributes);
                            }

                            File.WriteAllText(file, newText);
                            found.Add(file);
                            counterEdited++;
                        }
                        catch(Exception ex)
                        {
                            failed.Add(file);
                        }
                    }
                }
                counter++;
            }

            tpc.Dispose();
            File.WriteAllLines(Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location) + "\\found.txt", found);
            File.WriteAllLines(Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location) + "\\failed.txt", failed);

            Console.WriteLine("Press any key to exit...");
            Console.ReadKey();
        }

        private static FileAttributes RemoveAttribute(FileAttributes attributes, FileAttributes attributesToRemove)
        {
            return attributes & ~attributesToRemove;
        }

        private static Workspace FindWorkspaceByPath(TfsTeamProjectCollection tfs, string workspacePath)
        {
            VersionControlServer versionControl = tfs.GetService<VersionControlServer>();

            WorkspaceInfo workspaceInfo = Workstation.Current.GetLocalWorkspaceInfo(workspacePath);

            if (workspaceInfo != null)
            {
                return versionControl.GetWorkspace(workspaceInfo);
            }

            // No Workspace found using method 1, try to query all workspaces the user has on this machine.
            Workspace[] workspaces = versionControl.QueryWorkspaces(null, Environment.UserName, Environment.MachineName);
            foreach (Workspace w in workspaces)
            {
                foreach (WorkingFolder f in w.Folders)
                {
                    if (f.LocalItem.Equals(workspacePath))
                    {
                        return w;
                    }
                }
            }

            throw new Exception(String.Format("TFS Workspace cannot be determined for {0}.", workspacePath));
        }
    }
}

Answer 2

I recently found this happening in one of my solutions. It is nothing present in my code.

If I do a solution clean (right click on the solution -> Clean Solution) the problem goes away.