Accessibility overlay support for username field and scroll tracking (#700)

* Trigger overlay prompt when focusing on username field

* Adjust accessibility overlay position in response to scroll events

* Get username EditText with a single pass of the node tree, plus additional cleanup

src/Android/Accessibility/AccessibilityHelpers.cs

@@ -257,15 +257,48 @@ namespace Bit.Droid.Accessibility
             IEnumerable<AccessibilityNodeInfo> passwordNodes)
         {
             var allEditTexts = GetWindowNodes(root, e, n => EditText(n), false);
-            var usernameEditText = GetUsernameEditText(allEditTexts);
+            var usernameEditText = GetUsernameEditTextIfPasswordExists(allEditTexts);
             FillCredentials(usernameEditText, passwordNodes);
             allEditTexts.Dispose();
             usernameEditText = null;
         }
 
-        public static AccessibilityNodeInfo GetUsernameEditText(IEnumerable<AccessibilityNodeInfo> allEditTexts)
+        public static AccessibilityNodeInfo GetUsernameEditTextIfPasswordExists(
+            IEnumerable<AccessibilityNodeInfo> allEditTexts)
         {
-            return allEditTexts.TakeWhile(n => !n.Password).LastOrDefault();
+            AccessibilityNodeInfo previousEditText = null;
+            foreach(var editText in allEditTexts)
+            {
+                if(editText.Password)
+                {
+                    return previousEditText;
+                }
+                previousEditText = editText;
+            }
+            return null;
+        }
+
+        public static bool IsUsernameEditText(AccessibilityNodeInfo root, AccessibilityEvent e)
+        {
+            var allEditTexts = GetWindowNodes(root, e, n => EditText(n), false);
+            var usernameEditText = GetUsernameEditTextIfPasswordExists(allEditTexts);
+            if(usernameEditText != null)
+            { 
+                var isUsernameEditText = IsSameNode(usernameEditText, e.Source);
+                allEditTexts.Dispose();
+                usernameEditText = null;
+                return isUsernameEditText;
+            }
+            return false;
+        }
+
+        public static bool IsSameNode(AccessibilityNodeInfo info1, AccessibilityNodeInfo info2)
+        {
+            if(info1 != null && info2 != null) 
+            {
+                return info1.Equals(info2) || info1.GetHashCode() == info2.GetHashCode();
+            }
+            return false;
         }
 
         public static bool OverlayPermitted()
@@ -294,20 +327,59 @@ namespace Bit.Droid.Accessibility
             return view;
         }
 
-        public static Point GetOverlayAnchorPosition(AccessibilityNodeInfo root, AccessibilityEvent e)
+        public static WindowManagerLayoutParams GetOverlayLayoutParams()
+        {
+            WindowManagerTypes windowManagerType;
+            if(Build.VERSION.SdkInt >= BuildVersionCodes.O)
+            {
+                windowManagerType = WindowManagerTypes.ApplicationOverlay;
+            }
+            else
+            {
+                windowManagerType = WindowManagerTypes.Phone;
+            }
+
+            var layoutParams = new WindowManagerLayoutParams(
+                ViewGroup.LayoutParams.WrapContent,
+                ViewGroup.LayoutParams.WrapContent,
+                windowManagerType,
+                WindowManagerFlags.NotFocusable | WindowManagerFlags.NotTouchModal,
+                Format.Transparent);
+            layoutParams.Gravity = GravityFlags.Bottom | GravityFlags.Left;
+
+            return layoutParams;
+        }
+
+        public static Point GetOverlayAnchorPosition(AccessibilityNodeInfo root, AccessibilityNodeInfo anchorView)
         {
             var rootRect = new Rect();
             root.GetBoundsInScreen(rootRect);
             var rootRectHeight = rootRect.Height();
 
-            var eSrcRect = new Rect();
+            var anchorViewRect = new Rect();
-            e.Source.GetBoundsInScreen(eSrcRect);
+            anchorView.GetBoundsInScreen(anchorViewRect);
-            var eSrcRectLeft = eSrcRect.Left;
+            var anchorViewRectLeft = anchorViewRect.Left;
-            var eSrcRectTop = eSrcRect.Top;
+            var anchorViewRectTop = anchorViewRect.Top;
 
             var navBarHeight = GetNavigationBarHeight();
-            var calculatedTop = rootRectHeight - eSrcRectTop - navBarHeight;
+            var calculatedTop = rootRectHeight - anchorViewRectTop - navBarHeight;
-            return new Point(eSrcRectLeft, calculatedTop);
+            return new Point(anchorViewRectLeft, calculatedTop);
+        }
+
+        public static Point GetOverlayAnchorPosition(int nodeHash, AccessibilityNodeInfo root, AccessibilityEvent e)
+        {
+            Point point = null;
+            var allEditTexts = GetWindowNodes(root, e, n => EditText(n), false);
+            foreach(var node in allEditTexts)
+            {
+                if(node.GetHashCode() == nodeHash)
+                {
+                    point = GetOverlayAnchorPosition(root, node);
+                    break;
+                }
+            }
+            allEditTexts.Dispose();
+            return point;
         }
 
         private static int GetStatusBarHeight()

src/Android/Accessibility/AccessibilityService.cs

@@ -35,6 +35,8 @@ namespace Bit.Droid.Accessibility
 
         private IWindowManager _windowManager = null;
         private LinearLayout _overlayView = null;
+        private int _anchorViewHash = 0;
+        private int _lastAnchorX, _lastAnchorY = 0;
 
         public override void OnAccessibilityEvent(AccessibilityEvent e)
         {
@@ -68,33 +70,46 @@ namespace Bit.Droid.Accessibility
                 {
                     case EventTypes.ViewFocused:
                     case EventTypes.ViewClicked:
+                    case EventTypes.ViewScrolled:
                         var isKnownBroswer = AccessibilityHelpers.SupportedBrowsers.ContainsKey(root.PackageName);
                         if(e.EventType == EventTypes.ViewClicked && isKnownBroswer)
                         {
                             break;
                         }
-                        if(e.Source == null || !e.Source.Password)
+                        if(e.Source == null || e.PackageName == BitwardenPackage)
                         {
                             CancelOverlayPrompt();
                             break;
                         }
-                        if(e.PackageName == BitwardenPackage)
+                        if(e.EventType == EventTypes.ViewScrolled)
                         {
-                            CancelOverlayPrompt();
+                            AdjustOverlayForScroll(root, e);
                             break;
                         }
-                        if(ScanAndAutofill(root, e))
-                        {
-                            CancelOverlayPrompt();
-                        }
                         else
                         {
-                            OverlayPromptToAutofill(root, e);
+                            var isUsernameEditText1 = AccessibilityHelpers.IsUsernameEditText(root, e);
+                            var isPasswordEditText1 = e.Source?.Password ?? false;
+                            if(!isUsernameEditText1 && !isPasswordEditText1)
+                            {
+                                CancelOverlayPrompt();
+                                break;
+                            }
+                            if(ScanAndAutofill(root, e))
+                            {
+                                CancelOverlayPrompt();
+                            }
+                            else
+                            {
+                                OverlayPromptToAutofill(root, e);
+                            }
                         }
                         break;
                     case EventTypes.WindowContentChanged:
                     case EventTypes.WindowStateChanged:
-                        if(e.Source == null || e.Source.Password)
+                        var isUsernameEditText2 = AccessibilityHelpers.IsUsernameEditText(root, e);
+                        var isPasswordEditText2 = e.Source?.Password ?? false;
+                        if(e.Source == null || isUsernameEditText2 || isPasswordEditText2)
                         {
                             break;
                         }
@@ -188,7 +203,10 @@ namespace Bit.Droid.Accessibility
             System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Removed");
 
             _overlayView = null;
+            _anchorViewHash = 0;
             _lastNotificationUri = null;
+            _lastAnchorX = 0;
+            _lastAnchorY = 0;
         }
 
         private void OverlayPromptToAutofill(AccessibilityNodeInfo root, AccessibilityEvent e)
@@ -206,64 +224,78 @@ namespace Bit.Droid.Accessibility
                 return;
             }
 
-            WindowManagerTypes windowManagerType;
+            var layoutParams = AccessibilityHelpers.GetOverlayLayoutParams();
-            if(Build.VERSION.SdkInt >= BuildVersionCodes.O)
+            var anchorPosition = AccessibilityHelpers.GetOverlayAnchorPosition(root, e.Source);
-            {
-                windowManagerType = WindowManagerTypes.ApplicationOverlay;
-            }
-            else
-            {
-                windowManagerType = WindowManagerTypes.Phone;
-            }
-
-            var layoutParams = new WindowManagerLayoutParams(
-                ViewGroup.LayoutParams.WrapContent,
-                ViewGroup.LayoutParams.WrapContent,
-                windowManagerType,
-                WindowManagerFlags.NotFocusable | WindowManagerFlags.NotTouchModal,
-                Format.Transparent);
-
-            var anchorPosition = AccessibilityHelpers.GetOverlayAnchorPosition(root, e);
-
-            layoutParams.Gravity = GravityFlags.Bottom | GravityFlags.Left;
             layoutParams.X = anchorPosition.X;
             layoutParams.Y = anchorPosition.Y;
 
-            var intent = new Intent(this, typeof(AccessibilityActivity));
-            intent.PutExtra("uri", uri);
-            intent.SetFlags(ActivityFlags.NewTask | ActivityFlags.SingleTop | ActivityFlags.ClearTop);
-
             if(_windowManager == null)
             {
                 _windowManager = GetSystemService(WindowService).JavaCast<IWindowManager>();
             }
 
-            var updateView = false;
+            if(_overlayView == null)
-            if(_overlayView != null)
             {
-                updateView = true;
+                var intent = new Intent(this, typeof(AccessibilityActivity));
-            }
+                intent.PutExtra("uri", uri);
+                intent.SetFlags(ActivityFlags.NewTask | ActivityFlags.SingleTop | ActivityFlags.ClearTop);
 
-            _overlayView = AccessibilityHelpers.GetOverlayView(this);
+                _overlayView = AccessibilityHelpers.GetOverlayView(this);
-            _overlayView.Click += (sender, eventArgs) =>
+                _overlayView.Click += (sender, eventArgs) =>
-            {
+                {
-                CancelOverlayPrompt();
+                    CancelOverlayPrompt();
-                StartActivity(intent);
+                    StartActivity(intent);
-            };
+                };
 
-            _lastNotificationUri = uri;
+                _lastNotificationUri = uri;
 
-            if(updateView)
+                _windowManager.AddView(_overlayView, layoutParams);
-            {
+
-                _windowManager.UpdateViewLayout(_overlayView, layoutParams);
+                System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Added at X:{0} Y:{1}",
+                    layoutParams.X, layoutParams.Y);
             }
             else
             {
-                _windowManager.AddView(_overlayView, layoutParams);
+                _windowManager.UpdateViewLayout(_overlayView, layoutParams);
+
+                System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Updated to X:{0} Y:{1}",
+                    layoutParams.X, layoutParams.Y);
             }
 
-            System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View {0} X:{1} Y:{2}",
+            _anchorViewHash = e.Source.GetHashCode();
-                updateView ? "Updated to" : "Added at", layoutParams.X, layoutParams.Y);
+            _lastAnchorX = anchorPosition.X;
+            _lastAnchorY = anchorPosition.Y;
+        }
+
+        private void AdjustOverlayForScroll(AccessibilityNodeInfo root, AccessibilityEvent e)
+        {
+            if(_overlayView == null || _anchorViewHash <= 0)
+            {
+                return;
+            }
+
+            var anchorPosition = AccessibilityHelpers.GetOverlayAnchorPosition(_anchorViewHash, root, e);
+            if(anchorPosition == null)
+            {
+                return;
+            }
+
+            if(anchorPosition.X == _lastAnchorX && anchorPosition.Y == _lastAnchorY)
+            {
+                return;
+            }
+
+            var layoutParams = AccessibilityHelpers.GetOverlayLayoutParams();
+            layoutParams.X = anchorPosition.X;
+            layoutParams.Y = anchorPosition.Y;
+
+            _windowManager.UpdateViewLayout(_overlayView, layoutParams);
+
+            _lastAnchorX = anchorPosition.X;
+            _lastAnchorY = anchorPosition.Y;
+
+            System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Updated to X:{0} Y:{1}",
+                    layoutParams.X, layoutParams.Y);
         }
 
         private bool SkipPackage(string eventPackageName)

src/Android/Resources/xml/accessibilityservice.xml

@@ -2,7 +2,7 @@
 <accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
     android:summary="@string/AutoFillServiceSummary"
     android:description="@string/AutoFillServiceDescription"
-    android:accessibilityEventTypes="typeWindowStateChanged|typeWindowContentChanged|typeViewFocused|typeViewClicked"
+    android:accessibilityEventTypes="typeWindowStateChanged|typeWindowContentChanged|typeViewFocused|typeViewClicked|typeViewScrolled"
     android:accessibilityFeedbackType="feedbackGeneric"
     android:accessibilityFlags="flagReportViewIds"
     android:notificationTimeout="100"