Upstream/bitwarden-mobile
1
0
Fork 0
mirror of https://github.com/bitwarden/mobile.git synced 2024-12-26 16:57:59 +01:00
Code Issues Projects Releases Wiki Activity

Accessibility overlay support for username field and scroll tracking (#700)

Browse Source 
* Trigger overlay prompt when focusing on username field

* Adjust accessibility overlay position in response to scroll events

* Get username EditText with a single pass of the node tree, plus additional cleanup
This commit is contained in:
Matt Portune 2020-01-13 17:14:57 -05:00 committed by Kyle Spearrin
parent eb16025800
commit d0ba4b6702
3 changed files with 165 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
src/Android/Accessibility/AccessibilityHelpers.cs
View File

@ -257,15 +257,48 @@ namespace Bit.Droid.Accessibility
IEnumerable<AccessibilityNodeInfo> passwordNodes)
{
var allEditTexts = GetWindowNodes(root, e, n => EditText(n), false);
var usernameEditText = GetUsernameEditText(allEditTexts);
var usernameEditText = GetUsernameEditTextIfPasswordExists(allEditTexts);
FillCredentials(usernameEditText, passwordNodes);
allEditTexts.Dispose();
usernameEditText = null;
}
public static AccessibilityNodeInfo GetUsernameEditText(IEnumerable<AccessibilityNodeInfo> allEditTexts)
public static AccessibilityNodeInfo GetUsernameEditTextIfPasswordExists(
IEnumerable<AccessibilityNodeInfo> allEditTexts)
{
return allEditTexts.TakeWhile(n => !n.Password).LastOrDefault();
AccessibilityNodeInfo previousEditText = null;
foreach(var editText in allEditTexts)
{
if(editText.Password)
{
return previousEditText;
}
previousEditText = editText;
}
return null;
}
public static bool IsUsernameEditText(AccessibilityNodeInfo root, AccessibilityEvent e)
{
var allEditTexts = GetWindowNodes(root, e, n => EditText(n), false);
var usernameEditText = GetUsernameEditTextIfPasswordExists(allEditTexts);
if(usernameEditText != null)
{
var isUsernameEditText = IsSameNode(usernameEditText, e.Source);
allEditTexts.Dispose();
usernameEditText = null;
return isUsernameEditText;
}
return false;
}
public static bool IsSameNode(AccessibilityNodeInfo info1, AccessibilityNodeInfo info2)
{
if(info1 != null && info2 != null)
{
return info1.Equals(info2) || info1.GetHashCode() == info2.GetHashCode();
}
return false;
}
public static bool OverlayPermitted()
@ -294,20 +327,59 @@ namespace Bit.Droid.Accessibility
return view;
}
public static Point GetOverlayAnchorPosition(AccessibilityNodeInfo root, AccessibilityEvent e)
public static WindowManagerLayoutParams GetOverlayLayoutParams()
{
WindowManagerTypes windowManagerType;
if(Build.VERSION.SdkInt >= BuildVersionCodes.O)
{
windowManagerType = WindowManagerTypes.ApplicationOverlay;
}
else
{
windowManagerType = WindowManagerTypes.Phone;
}
var layoutParams = new WindowManagerLayoutParams(
ViewGroup.LayoutParams.WrapContent,
ViewGroup.LayoutParams.WrapContent,
windowManagerType,
WindowManagerFlags.NotFocusable | WindowManagerFlags.NotTouchModal,
Format.Transparent);
layoutParams.Gravity = GravityFlags.Bottom | GravityFlags.Left;
return layoutParams;
}
public static Point GetOverlayAnchorPosition(AccessibilityNodeInfo root, AccessibilityNodeInfo anchorView)
{
var rootRect = new Rect();
root.GetBoundsInScreen(rootRect);
var rootRectHeight = rootRect.Height();
var eSrcRect = new Rect();
e.Source.GetBoundsInScreen(eSrcRect);
var eSrcRectLeft = eSrcRect.Left;
var eSrcRectTop = eSrcRect.Top;
var anchorViewRect = new Rect();
anchorView.GetBoundsInScreen(anchorViewRect);
var anchorViewRectLeft = anchorViewRect.Left;
var anchorViewRectTop = anchorViewRect.Top;
var navBarHeight = GetNavigationBarHeight();
var calculatedTop = rootRectHeight - eSrcRectTop - navBarHeight;
return new Point(eSrcRectLeft, calculatedTop);
var calculatedTop = rootRectHeight - anchorViewRectTop - navBarHeight;
return new Point(anchorViewRectLeft, calculatedTop);
}
public static Point GetOverlayAnchorPosition(int nodeHash, AccessibilityNodeInfo root, AccessibilityEvent e)
{
Point point = null;
var allEditTexts = GetWindowNodes(root, e, n => EditText(n), false);
foreach(var node in allEditTexts)
{
if(node.GetHashCode() == nodeHash)
{
point = GetOverlayAnchorPosition(root, node);
break;
}
}
allEditTexts.Dispose();
return point;
}
private static int GetStatusBarHeight()

132
src/Android/Accessibility/AccessibilityService.cs
View File

@ -35,6 +35,8 @@ namespace Bit.Droid.Accessibility
private IWindowManager _windowManager = null;
private LinearLayout _overlayView = null;
private int _anchorViewHash = 0;
private int _lastAnchorX, _lastAnchorY = 0;
public override void OnAccessibilityEvent(AccessibilityEvent e)
{
@ -68,33 +70,46 @@ namespace Bit.Droid.Accessibility
{
case EventTypes.ViewFocused:
case EventTypes.ViewClicked:
case EventTypes.ViewScrolled:
var isKnownBroswer = AccessibilityHelpers.SupportedBrowsers.ContainsKey(root.PackageName);
if(e.EventType == EventTypes.ViewClicked && isKnownBroswer)
{
break;
}
if(e.Source == null || !e.Source.Password)
if(e.Source == null || e.PackageName == BitwardenPackage)
{
CancelOverlayPrompt();
break;
}
if(e.PackageName == BitwardenPackage)
if(e.EventType == EventTypes.ViewScrolled)
{
CancelOverlayPrompt();
AdjustOverlayForScroll(root, e);
break;
}
if(ScanAndAutofill(root, e))
{
CancelOverlayPrompt();
}
else
{
OverlayPromptToAutofill(root, e);
var isUsernameEditText1 = AccessibilityHelpers.IsUsernameEditText(root, e);
var isPasswordEditText1 = e.Source?.Password ?? false;
if(!isUsernameEditText1 && !isPasswordEditText1)
{
CancelOverlayPrompt();
break;
}
if(ScanAndAutofill(root, e))
{
CancelOverlayPrompt();
}
else
{
OverlayPromptToAutofill(root, e);
}
}
break;
case EventTypes.WindowContentChanged:
case EventTypes.WindowStateChanged:
if(e.Source == null || e.Source.Password)
var isUsernameEditText2 = AccessibilityHelpers.IsUsernameEditText(root, e);
var isPasswordEditText2 = e.Source?.Password ?? false;
if(e.Source == null || isUsernameEditText2 || isPasswordEditText2)
{
break;
}
@ -188,7 +203,10 @@ namespace Bit.Droid.Accessibility
System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Removed");
_overlayView = null;
_anchorViewHash = 0;
_lastNotificationUri = null;
_lastAnchorX = 0;
_lastAnchorY = 0;
}
private void OverlayPromptToAutofill(AccessibilityNodeInfo root, AccessibilityEvent e)
@ -206,64 +224,78 @@ namespace Bit.Droid.Accessibility
return;
}
WindowManagerTypes windowManagerType;
if(Build.VERSION.SdkInt >= BuildVersionCodes.O)
{
windowManagerType = WindowManagerTypes.ApplicationOverlay;
}
else
{
windowManagerType = WindowManagerTypes.Phone;
}
var layoutParams = new WindowManagerLayoutParams(
ViewGroup.LayoutParams.WrapContent,
ViewGroup.LayoutParams.WrapContent,
windowManagerType,
WindowManagerFlags.NotFocusable | WindowManagerFlags.NotTouchModal,
Format.Transparent);
var anchorPosition = AccessibilityHelpers.GetOverlayAnchorPosition(root, e);
layoutParams.Gravity = GravityFlags.Bottom | GravityFlags.Left;
var layoutParams = AccessibilityHelpers.GetOverlayLayoutParams();
var anchorPosition = AccessibilityHelpers.GetOverlayAnchorPosition(root, e.Source);
layoutParams.X = anchorPosition.X;
layoutParams.Y = anchorPosition.Y;
var intent = new Intent(this, typeof(AccessibilityActivity));
intent.PutExtra("uri", uri);
intent.SetFlags(ActivityFlags.NewTask | ActivityFlags.SingleTop | ActivityFlags.ClearTop);
if(_windowManager == null)
{
_windowManager = GetSystemService(WindowService).JavaCast<IWindowManager>();
}
var updateView = false;
if(_overlayView != null)
if(_overlayView == null)
{
updateView = true;
}
var intent = new Intent(this, typeof(AccessibilityActivity));
intent.PutExtra("uri", uri);
intent.SetFlags(ActivityFlags.NewTask | ActivityFlags.SingleTop | ActivityFlags.ClearTop);
_overlayView = AccessibilityHelpers.GetOverlayView(this);
_overlayView.Click += (sender, eventArgs) =>
{
CancelOverlayPrompt();
StartActivity(intent);
};
_overlayView = AccessibilityHelpers.GetOverlayView(this);
_overlayView.Click += (sender, eventArgs) =>
{
CancelOverlayPrompt();
StartActivity(intent);
};
_lastNotificationUri = uri;
_lastNotificationUri = uri;
if(updateView)
{
_windowManager.UpdateViewLayout(_overlayView, layoutParams);
_windowManager.AddView(_overlayView, layoutParams);
System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Added at X:{0} Y:{1}",
layoutParams.X, layoutParams.Y);
}
else
{
_windowManager.AddView(_overlayView, layoutParams);
_windowManager.UpdateViewLayout(_overlayView, layoutParams);
System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Updated to X:{0} Y:{1}",
layoutParams.X, layoutParams.Y);
}
System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View {0} X:{1} Y:{2}",
updateView ? "Updated to" : "Added at", layoutParams.X, layoutParams.Y);
_anchorViewHash = e.Source.GetHashCode();
_lastAnchorX = anchorPosition.X;
_lastAnchorY = anchorPosition.Y;
}
private void AdjustOverlayForScroll(AccessibilityNodeInfo root, AccessibilityEvent e)
{
if(_overlayView == null || _anchorViewHash <= 0)
{
return;
}
var anchorPosition = AccessibilityHelpers.GetOverlayAnchorPosition(_anchorViewHash, root, e);
if(anchorPosition == null)
{
return;
}
if(anchorPosition.X == _lastAnchorX && anchorPosition.Y == _lastAnchorY)
{
return;
}
var layoutParams = AccessibilityHelpers.GetOverlayLayoutParams();
layoutParams.X = anchorPosition.X;
layoutParams.Y = anchorPosition.Y;
_windowManager.UpdateViewLayout(_overlayView, layoutParams);
_lastAnchorX = anchorPosition.X;
_lastAnchorY = anchorPosition.Y;
System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Updated to X:{0} Y:{1}",
layoutParams.X, layoutParams.Y);
}
private bool SkipPackage(string eventPackageName)

2
src/Android/Resources/xml/accessibilityservice.xml
View File

@ -2,7 +2,7 @@
<accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
android:summary="@string/AutoFillServiceSummary"
android:description="@string/AutoFillServiceDescription"
android:accessibilityEventTypes="typeWindowStateChanged|typeWindowContentChanged|typeViewFocused|typeViewClicked"
android:accessibilityEventTypes="typeWindowStateChanged|typeWindowContentChanged|typeViewFocused|typeViewClicked|typeViewScrolled"
android:accessibilityFeedbackType="feedbackGeneric"
android:accessibilityFlags="flagReportViewIds"
android:notificationTimeout="100"